You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by li...@apache.org on 2026/08/13 08:23:06 UTC
(rocketmq-flink) branch main updated: [ISSUE #135] feat: add gRPC connector on LiteSimpleConsumer with downstream ack and sub-topic throttling (#136)
This is an automated email from the ASF dual-hosted git repository.
lizhimins pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/rocketmq-flink.git
The following commit(s) were added to refs/heads/main by this push:
new 6f8b770 [ISSUE #135] feat: add gRPC connector on LiteSimpleConsumer with downstream ack and sub-topic throttling (#136)
6f8b770 is described below
commit 6f8b77023286bcd1e8f5f5eba8d872baee287488
Author: lizhimins <70...@qq.com>
AuthorDate: Thu Aug 13 16:23:02 2026 +0800
[ISSUE #135] feat: add gRPC connector on LiteSimpleConsumer with downstream ack and sub-topic throttling (#136)
Add the flink-connector-rocketmq-grpc module built on rocketmq-client-java
5.2.1+ LiteSimpleConsumer (Pop model):
- Source binds one main lite topic with a wildcard subscription and never
acks; each subtask shares one consumer across fetch-concurrency long-poll
workers.
- Emits AckableMessage<OUT> carrying a serializable, credential-free
RocketMQReceiptHandle that can cross shuffle boundaries.
- Downstream ack via a per-TaskManager reference-counted client pool, with
two wirings: RocketMQAckProcessFunction (direct ack) and
MessageThrottlePolicy + RocketMQThrottleProcessFunction (policy-based
fair throttling with DLQ safety valves).
- Optional invisible-duration renewal prevents premature redelivery under
backpressure.
- gRPC producer sink and rocketmq-grpc SQL connector; design documented in
docs/grpc-connector.md.
---
README.md | 19 +-
docs/grpc-connector.md | 241 +++++++++++-
docs/sql-connector.md | 4 +-
flink-connector-rocketmq-grpc/pom.xml | 153 ++++++++
.../rocketmq/grpc/RocketMQGrpcOptions.java | 105 ++++++
.../rocketmq/grpc/ack/AckableMessage.java | 75 ++++
.../rocketmq/grpc/ack/AckableMessageTypeInfo.java | 264 ++++++++++++++
.../rocketmq/grpc/ack/MessageThrottlePolicy.java | 46 +++
.../grpc/ack/RocketMQAckProcessFunction.java | 115 ++++++
.../rocketmq/grpc/ack/RocketMQLiteAckClient.java | 398 ++++++++++++++++++++
.../rocketmq/grpc/ack/RocketMQReceiptHandle.java | 299 +++++++++++++++
.../grpc/ack/RocketMQReceiptHandleCodec.java | 162 +++++++++
.../grpc/ack/RocketMQThrottleProcessFunction.java | 104 ++++++
.../grpc/common/ClientConfigurationProvider.java | 79 ++++
.../rocketmq/grpc/common/CredentialsResolver.java | 67 ++++
.../rocketmq/grpc/common/CredentialsResolvers.java | 99 +++++
.../grpc/common/FilterExpressionParser.java | 64 ++++
.../rocketmq/grpc/sink/ProducerProvider.java | 51 +++
.../rocketmq/grpc/sink/RocketMQGrpcSink.java | 70 ++++
.../grpc/sink/RocketMQGrpcSinkBuilder.java | 99 +++++
.../grpc/sink/RocketMQGrpcSinkOptions.java | 68 ++++
.../rocketmq/grpc/sink/RocketMQGrpcSinkWriter.java | 83 +++++
.../RocketMQGrpcSerializationSchema.java | 68 ++++
.../RocketMQGrpcSerializationSchemaWrapper.java | 64 ++++
.../source/InvisibleDurationRenewalPolicies.java | 69 ++++
.../source/InvisibleDurationRenewalPolicy.java | 69 ++++
.../rocketmq/grpc/source/RocketMQGrpcSource.java | 163 +++++++++
.../grpc/source/RocketMQGrpcSourceBuilder.java | 144 ++++++++
.../grpc/source/RocketMQGrpcSourceOptions.java | 118 ++++++
.../RocketMQGrpcDeserializationSchema.java | 62 ++++
.../RocketMQGrpcDeserializationSchemaWrapper.java | 62 ++++
.../enumerator/RocketMQGrpcSourceEnumState.java | 28 ++
.../RocketMQGrpcSourceEnumStateSerializer.java | 46 +++
.../enumerator/RocketMQGrpcSourceEnumerator.java | 86 +++++
.../source/reader/LiteSimpleConsumerProvider.java | 58 +++
.../rocketmq/grpc/source/reader/MessageView.java | 54 +++
.../grpc/source/reader/MessageViewImpl.java | 128 +++++++
.../reader/RocketMQGrpcSourceFetcherManager.java | 45 +++
.../source/reader/RocketMQGrpcSourceReader.java | 78 ++++
.../reader/RocketMQGrpcSourceRecordEmitter.java | 99 +++++
.../reader/RocketMQGrpcSourceSplitReader.java | 325 +++++++++++++++++
.../grpc/source/split/RocketMQGrpcSourceSplit.java | 65 ++++
.../split/RocketMQGrpcSourceSplitSerializer.java | 46 +++
.../source/split/RocketMQGrpcSourceSplitState.java | 48 +++
.../grpc/table/RocketMQGrpcConnectorOptions.java | 188 ++++++++++
.../table/RocketMQGrpcDynamicTableFactory.java | 245 +++++++++++++
.../grpc/table/RocketMQGrpcDynamicTableSink.java | 109 ++++++
.../grpc/table/RocketMQGrpcDynamicTableSource.java | 197 ++++++++++
.../grpc/table/RocketMQGrpcReadableMetadata.java | 118 ++++++
.../grpc/table/RocketMQGrpcRowDataConverter.java | 181 +++++++++
.../org.apache.flink.table.factories.Factory | 16 +
.../rocketmq/grpc/RocketMQGrpcITCase.java | 405 +++++++++++++++++++++
.../rocketmq/grpc/RocketMQGrpcLiteE2EVerify.java | 346 ++++++++++++++++++
.../grpc/ack/AckableMessageSerializerTest.java | 75 ++++
.../grpc/ack/RocketMQLiteAckClientTest.java | 82 +++++
.../grpc/ack/RocketMQReceiptHandleCodecTest.java | 110 ++++++
.../ack/RocketMQReceiptHandleSerializerTest.java | 73 ++++
.../ack/RocketMQThrottleProcessFunctionTest.java | 149 ++++++++
.../grpc/common/CredentialsResolversTest.java | 153 ++++++++
.../grpc/sink/RocketMQGrpcSinkBuilderTest.java | 90 +++++
.../InvisibleDurationRenewalPoliciesTest.java | 95 +++++
.../grpc/source/RocketMQGrpcSourceBuilderTest.java | 94 +++++
.../RocketMQGrpcSourceEnumStateSerializerTest.java | 42 +++
.../RocketMQGrpcSourceSplitSerializerTest.java | 44 +++
.../grpc/source/reader/MessageViewImplTest.java | 141 +++++++
.../table/RocketMQGrpcDynamicTableFactoryTest.java | 83 +++++
.../table/RocketMQGrpcRowDataConverterTest.java | 172 +++++++++
.../common/config/RocketMQConfigValidator.java | 6 +-
.../connector/rocketmq/sink/RocketMQSink.java | 2 +-
.../rocketmq/sink/RocketMQSinkBuilder.java | 2 +-
.../rocketmq/sink/RocketMQSinkOptions.java | 4 +-
.../rocketmq/sink/writer/RocketMQWriter.java | 2 +-
.../rocketmq/source/InnerConsumerImpl.java | 14 +-
.../connector/rocketmq/source/RocketMQSource.java | 1 -
.../rocketmq/source/RocketMQSourceOptions.java | 2 +-
.../source/enumerator/offset/OffsetsSelector.java | 2 +-
.../reader/RocketMQSourceFetcherManager.java | 2 +-
.../RocketMQRowDeserializationSchema.java | 3 +-
.../connectors/rocketmq/RocketMQSink.java | 4 +-
.../rocketmq/RocketMQSourceFunction.java | 26 +-
.../rocketmq/table/RocketMQDynamicTableSink.java | 2 +-
.../table/RocketMQDynamicTableSourceFactory.java | 2 +-
.../table/RocketMQRowDataSerializationSchema.java | 4 +-
.../rocketmq/table/RocketMQRowDataSink.java | 2 +-
.../rocketmq/table/RocketMQScanTableSource.java | 8 +-
.../rocketmq/catalog/RocketMQCatalogTest.java | 15 +-
.../rocketmq/example/ConnectorIntegrationTest.java | 19 +-
.../rocketmq/example/SqlIntegrationTest.java | 3 +-
.../example/TransactionSinkIntegrationTest.java | 4 +-
.../rocketmq/sink/InnerProducerImplTest.java | 3 +-
.../connectors/rocketmq/RocketMQSourceTest.java | 2 +-
.../rocketmq/example/LegacyConnectorExample.java | 12 +-
...ocketMQDynamicTableSourceFactoryOffsetTest.java | 1 -
.../table/RocketMQScanTableSourceFilterTest.java | 6 +-
.../table/RocketMQScanTableSourceTest.java | 1 -
flink-sql-connector-rocketmq-grpc/pom.xml | 144 ++++++++
pom.xml | 2 +
97 files changed, 7995 insertions(+), 103 deletions(-)
diff --git a/README.md b/README.md
index 37084b2..a2a91d4 100644
--- a/README.md
+++ b/README.md
@@ -11,8 +11,10 @@ Two connector tracks are shipped:
| Track | Module | RocketMQ client | Best for |
| --- | --- | --- | --- |
| **Remoting** | `flink-connector-rocketmq` | `rocketmq-client` (remoting protocol) | Classic topics, FLIP-27 source + SinkV2 sink, SQL |
+| **gRPC** | `flink-connector-rocketmq-grpc` | `rocketmq-client-java` (gRPC protocol) | RocketMQ 5.x lite topics, downstream ack / fair throttling |
-The SQL fat-jar is packaged by `flink-sql-connector-rocketmq`.
+The SQL fat-jars are packaged by `flink-sql-connector-rocketmq` and
+`flink-sql-connector-rocketmq-grpc`.
## Apache Flink
@@ -93,6 +95,20 @@ CREATE TABLE rocketmq_source (
);
```
+### gRPC lite consumer (RocketMQ 5.x)
+
+```java
+RocketMQGrpcSource<String> source = RocketMQGrpcSource.<String>builder()
+ .setEndpoints("127.0.0.1:8081")
+ .setConsumerGroup("GID-lite")
+ .setMainTopic("LiteMainTopic") // wildcard-subscribes all lite topics under it
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build();
+```
+
+The gRPC source emits `AckableMessage<OUT>` and never acks by itself: the downstream operators own
+the ack / throttle decision. See the gRPC connector doc for the ack operator wirings.
+
## Documentation
Connector documentation is located in the `docs/` directory of this repository:
@@ -100,6 +116,7 @@ Connector documentation is located in the `docs/` directory of this repository:
| Document | Content |
| --- | --- |
| [docs/remoting-connector.md](docs/remoting-connector.md) | Remoting DataStream connector: usage and all configuration options |
+| [docs/grpc-connector.md](docs/grpc-connector.md) | gRPC lite connector: prerequisites, downstream ack / throttling, options |
| [docs/sql-connector.md](docs/sql-connector.md) | Table/SQL connector: DDL, metadata columns, fat-jar notes |
| [docs/legacy-connector.md](docs/legacy-connector.md) | Legacy `RocketMQSourceFunction` / `RocketMQSink` (deprecated) |
| [docs/connector-overview.md](docs/connector-overview.md) | Feature comparison with other messaging connectors |
diff --git a/docs/grpc-connector.md b/docs/grpc-connector.md
index a799a67..017b176 100644
--- a/docs/grpc-connector.md
+++ b/docs/grpc-connector.md
@@ -4,7 +4,215 @@ The gRPC connector (`flink-connector-rocketmq-grpc`) targets RocketMQ 5.x **lite
`rocketmq-client-java` 5.2.1+. The source binds one main lite topic with a wildcard subscription,
never acks by itself, and hands the ack / throttle decision to downstream operators.
-## Prerequisites
+This document covers both the design (architecture, semantics, decisions and trade-offs) and the
+usage of the connector.
+
+---
+
+## 1. Motivation
+
+The target scenario is a **main lite topic with a large number of sub (lite) topics** underneath,
+e.g. AI workloads where every tenant/session gets its own sub topic:
+
+- The source subscribes to the main topic with a **wildcard (generalized) subscription**; the
+ broker funnels messages of *all* sub topics through one `receive()` stream, so the reader never
+ has to enumerate or track sub topics.
+- Whether a message should be **acknowledged** or **deferred (throttled)** is a business decision
+ that is only known *after* downstream processing — possibly on the other side of a
+ keyBy/shuffle. A hot sub topic that consumes too many resources should be slowed down without
+ starving the other sub topics.
+
+This leads to the two defining properties of the connector:
+
+1. **The source never acknowledges messages.** Acknowledgement is deferred to downstream
+ operators, which receive a self-contained, serializable receipt handle with every record.
+2. **Throttling is expressed per message via `changeInvisibleDuration`.** Deferring the
+ redelivery of a hot sub topic's messages naturally yields fair sharing of the fetch budget:
+
+```
+ sub topic t_hot floods messages ─┐
+ ├─► each one gets changeInvisibleDuration(+20s)
+ │ → invisible to the group for the next 20s
+ ▼
+ reader.receive() only returns currently-visible messages:
+ ├─ t1, t2, t3 ... normal messages (visible) ← keep flowing
+ └─ t_hot messages mostly invisible ← suppressed, trickle back on expiry
+
+ ⇒ the hot sub topic is throttled, the others are not starved = fairness
+```
+
+## 2. Consumption model
+
+RocketMQ 5.x offers a push client (`PushConsumer`) and pull/Pop clients
+(`SimpleConsumer` / `LiteSimpleConsumer`). The connector uses **`LiteSimpleConsumer`**
+(available since `rocketmq-client-java` 5.2.1), whose consumption is split into three explicit
+phases that map cleanly onto a Flink pipeline:
+
+1. **Receive** — `receive(maxMessageNum, invisibleDuration)` long-polls the proxy; returned
+ messages become *invisible* to the rest of the consumer group for `invisibleDuration`.
+2. **Process** — the caller (here: the Flink job) processes the messages.
+3. **Ack / changeInvisibleDuration** — `ack()` commits a message;
+ `changeInvisibleDuration()` extends/renews its invisibility. **Un-acked messages are
+ redelivered by the broker once their invisible duration expires** — this is the source of the
+ at-least-once guarantee.
+
+Key SDK facts the design relies on (verified against the SDK sources):
+
+- **Wildcard subscription = `bindTopic` only.** Binding the main topic without calling
+ `subscribeLite` puts the consumer into generalized mode: it receives from all sub topics.
+ The consumer group must carry the broker-side attribute `lite.sub.wildcard=true`
+ (a **consumer group** attribute — see Prerequisites).
+- **Pop receipt handles are self-contained.** The server-side handle encodes its own routing
+ (broker, queue, offsets, timestamps) as a string; the proxy routes an ack by the handle alone.
+ Any consumer of the **same group** can ack a message, regardless of which consumer received it.
+- **`SimpleConsumer`/`LiteSimpleConsumer` are thread-safe.** Multiple threads may block in
+ `receive()` on one consumer instance concurrently.
+- **`OffsetOption` (start-position control) exists only for exact `subscribeLite`**, therefore
+ the wildcard-mode source has no startup-offset capability.
+
+## 3. Architecture
+
+### 3.1 Source reader
+
+Because the broker performs **message-level load balancing** across the consumer group, the
+source does not partition topics into splits. Every subtask binds the same main topic and simply
+receives whatever the broker hands out; a single placeholder split only triggers the reader to
+start. Per subtask:
+
+```
+subtask
+ └─ SourceReader (mailbox thread)
+ └─ SingleThreadFetcherManager → 1 fetcher thread
+ └─ SplitReader
+ ├─ 1 shared LiteSimpleConsumer (thread-safe)
+ ├─ N ReceiveWorker threads (N = rocketmq.source.fetch-concurrency)
+ │ loop: consumer.receive(maxMessageNum, invisibleDuration)
+ │ → put into a bounded in-memory queue (capacity = maxMessageNum × N)
+ └─ fetch(): drain the queue in batches for the record emitter
+```
+
+- **One consumer, many workers.** A blocking `receive()` long poll only occupies its calling
+ thread, so `fetch-concurrency` worker threads sharing one consumer keep that many long polls in
+ flight — raising a single subtask's throughput without extra clientIds, heartbeats, connections
+ or route caches. Total throughput scales with
+ `parallelism × fetch-concurrency × maxMessageNum / receive-latency`.
+- **Backpressure = stop receiving.** When the bounded queue is full, workers block on `put()`;
+ the pull model has no explicit pause API and does not need one.
+- **Receive failures back off exponentially** (100 ms doubling up to 30 s) instead of
+ hot-looping.
+
+### 3.2 Receipt handle and record type
+
+The source emits `AckableMessage<OUT> = { value, RocketMQReceiptHandle }`.
+
+`RocketMQReceiptHandle` is a small, immutable, `Serializable` value object made of strings and an
+int:
+
+| Field group | Fields | Purpose |
+| --- | --- | --- |
+| Routing triple | `endpoint`, `namespace`, `consumerGroup` | Selects which pooled consumer must issue the ack RPC. Records from multiple sources (different clusters/namespaces/groups) can coexist in one stream. |
+| Message identity | `topic`, `liteTopic`, `messageId`, `receiptHandle`, `deliveryAttempt` | Rebuilds the minimal SDK message view required by `ack` / `changeInvisibleDuration`. |
+
+It deliberately contains **no credentials, no protobuf blob and no message body**, so it is safe
+and cheap to ship across keyBy/shuffle boundaries. Dedicated `TypeInformation`/`TypeSerializer`
+implementations (`AckableMessageTypeInfo` and the nested serializers) keep the type off the Kryo
+fallback path and give it a stable wire format.
+
+The single place that touches SDK-internal classes (`org.apache.rocketmq.client.java.*`) is
+`RocketMQReceiptHandleCodec`: the public `apis` package exposes no receipt handle, so extracting
+it on the source side and rebuilding a minimal message view on the ack side requires internal
+types. Confining that (experimental) coupling to one `@Internal` adapter keeps the rest of the
+connector SDK-clean.
+
+### 3.3 Downstream acknowledgement
+
+Downstream operators obtain a shared, credential-free ack client:
+
+- **Pooling.** `RocketMQLiteAckClient` is shared per TaskManager JVM via reference-counted
+ `acquire`/`release` keyed by the client configuration. Internally it lazily keeps one
+ same-group `LiteSimpleConsumer` per routing triple `(endpoint, namespace, consumerGroup)`
+ carried by the incoming handles. The consumer built for a handle uses the *handle's* endpoints
+ and namespace (the SDK stamps ack requests with the consumer's namespace) and the *operator
+ configuration's* credentials/TLS/timeout.
+- **Retry & tolerance.** Ack RPCs retry with bounded exponential backoff. An
+ `INVALID_RECEIPT_HANDLE` response (expired handle) is tolerated as a warning: the message will
+ simply be redelivered, which is preferable to failing the job under at-least-once semantics.
+- **Credentials never travel in the stream.** They are configured on the ack operator and
+ resolved locally on each TaskManager — either static access/secret keys, or a pluggable
+ per-endpoint `CredentialsResolver` (e.g. environment variables, mounted secrets, external KMS)
+ that also keeps plaintext secrets out of the job graph.
+
+Two wirings are provided on top of the client (DataStream API only):
+
+1. **Client injection** — extend the abstract `RocketMQAckProcessFunction` and call the
+ protected `ack()` / `changeInvisibleDuration()` from business logic. No extra edge in the job
+ graph, lowest latency.
+2. **Policy-based throttling** — implement `MessageThrottlePolicy<T>`
+ (`Optional<Duration> onMessage(value)`: empty = ack, duration = defer) and run it with the
+ one-stop `RocketMQThrottleProcessFunction<T>`.
+
+### 3.4 Throttling semantics and safety valves
+
+Using `changeInvisibleDuration` for throttling is a deliberate, documented trade-off: it is a
+**per-message defer**, not a topic-level switch. Consequences and mitigations:
+
+- Every deferred message costs one extra RPC, and deferring reorders the stream — downstream must
+ already be idempotent and order-tolerant under at-least-once.
+- Each redelivery increments `deliveryAttempt`; endless deferring would eventually push a message
+ into the dead-letter queue. `RocketMQThrottleProcessFunction` therefore enforces two safety
+ valves: once `deliveryAttempt` reaches `maxDeliveryAttempt` (default 16) the message is acked
+ instead of deferred, and every requested delay is capped at `maxInvisibleDuration`
+ (default 30 min).
+- The broker slows down dispatch for lite topics with a large un-acked backlog; spreading load
+ over many sub topics is part of the intended usage pattern.
+
+### 3.5 Invisible-duration renewal
+
+Under backpressure, messages can sit in the reader's internal queue long enough for their
+invisible duration to expire, causing spurious redelivery. An optional
+`InvisibleDurationRenewalPolicy` renews queue-resident messages `renewal-ahead-time` before
+expiry. Messages are **frozen once emitted** to the record emitter: renewing refreshes the
+receipt handle, which would invalidate the handle already travelling downstream.
+
+## 4. Design decisions and alternatives
+
+**Why downstream ack at all?** Mainstream Flink connectors with a Pop/lease-like model (GCP
+Pub/Sub, Amazon SQS) ack **on the source side**, typically on `notifyCheckpointComplete`. That
+only works when the ack decision is available in the source subtask. Here the requirement is to
+decide *after* arbitrary downstream processing (possibly across shuffles), so the handle must
+travel with the record and the ack must happen downstream.
+
+Alternatives that were evaluated and rejected:
+
+| Alternative | Why rejected |
+| --- | --- |
+| Source-side ack on checkpoint (Pub/Sub / SQS style) | Decision must be co-located with the source subtask; does not meet the requirement. |
+| Feeding ack commands back to the reader | Flink's DAG is acyclic: DataStream Iterations are deprecated and checkpoint-incompatible, `OperatorCoordinator` does not connect different operators, and an external command topic would need credentials downstream anyway. |
+| Carrying credentials in the record stream | Security red line. Credentials stay in operator configuration, resolved locally per TaskManager. |
+| Serializing the SDK protobuf receipt blob | Unnecessary — the server-side handle string self-encodes routing; a handful of strings suffice. |
+
+Other decisions:
+
+- **Commit point is user-driven.** The connector does not align acks with checkpoints; the user
+ acks when processing is complete. This gives at-least-once with user-controlled granularity.
+- **One main topic per source** (SDK `bindTopic` limitation). Use multiple sources for multiple
+ main topics; the routing triple in the handle lets one downstream ack operator serve all of
+ them.
+- **Ack/throttle APIs are DataStream-only.** The SQL/Table path consumes values only.
+
+## 5. Delivery semantics
+
+- **At-least-once.** The source never acks; un-acked messages are redelivered after
+ `invisible-duration` expires. Downstream must be idempotent.
+- `invisible-duration` must cover the full downstream processing time of a message (including
+ shuffles and slow operators such as model inference); use the renewal policy when queueing
+ time is unpredictable.
+- Exactly-once is out of scope for the Pop model: there is no offset the connector could
+ checkpoint-align, and acks are user-driven by design.
+
+---
+
+## 6. Prerequisites
- RocketMQ 5.x cluster with lite topic support enabled on the broker
(`enableMultiDispatch`, `enableLmq`, etc. — preset by the official Helm chart).
@@ -19,7 +227,7 @@ mqadmin updateSubGroup -n <ns:9876> -c <cluster> -g GID-lite \
`lite.sub.wildcard` is a **consumer group** attribute, not a topic attribute. Without it the
wildcard subscription receives nothing.
-## Source
+## 7. Source
```java
RocketMQGrpcSource<String> source = RocketMQGrpcSource.<String>builder()
@@ -41,13 +249,11 @@ handle can cross keyBy/shuffle boundaries; credentials never travel with it.
### Downstream ack / throttling
Unacked messages are redelivered by the broker after `invisible-duration` expires
-(at-least-once — downstream must be idempotent). Three wirings are provided:
+(at-least-once — downstream must be idempotent). Two wirings are provided:
-1. **Standalone ack operator** — user operators emit `AckCommand{handle, action, delay}`,
- `RocketMQAckOperator` executes them. Users never touch the SDK.
-2. **Client injection** — extend `RocketMQAckProcessFunction` and call `ack()` /
+1. **Client injection** — extend `RocketMQAckProcessFunction` and call `ack()` /
`changeInvisibleDuration()` directly.
-3. **Policy-based throttling** — implement `MessageThrottlePolicy<T>`
+2. **Policy-based throttling** — implement `MessageThrottlePolicy<T>`
(`Optional<Duration> onMessage(value)`: empty = ack, duration = defer via
`changeInvisibleDuration`) and run it with `RocketMQThrottleProcessFunction<T>`.
Built-in safety valves: maxDeliveryAttempt=16, maxInvisibleDuration=30min (avoids DLQ from
@@ -67,7 +273,7 @@ builder.setRenewalPolicyClass("com.example.MyRenewalPolicy")
.setRenewalAheadTime(Duration.ofSeconds(5));
```
-## Sink
+## 8. Sink
```java
RocketMQGrpcSink<String> sink = RocketMQGrpcSink.<String>builder()
@@ -78,7 +284,7 @@ RocketMQGrpcSink<String> sink = RocketMQGrpcSink.<String>builder()
.build();
```
-## Options
+## 9. Options
### Client (`rocketmq.client.*`)
@@ -98,7 +304,7 @@ RocketMQGrpcSink<String> sink = RocketMQGrpcSink.<String>builder()
| `rocketmq.source.main-topic` | String | (none) | Main lite topic to bind, required |
| `rocketmq.source.consumer-group` | String | (none) | Consumer group, required |
| `rocketmq.source.fetch-concurrency` | Integer | 1 | Concurrent long-polling receive workers |
-| `rocketmq.source.await-duration` | Duration | 30s | Long-polling await time |
+| `rocketmq.source.await-duration` | Duration | 20s | Long-polling await time |
| `rocketmq.source.invisible-duration` | Duration | 60s | Invisible time per receive (min 10s) |
| `rocketmq.source.max-message-num` | Integer | 32 | Max messages per receive |
| `rocketmq.source.renewal-policy-class` | String | (none) | `InvisibleDurationRenewalPolicy` implementation |
@@ -112,8 +318,19 @@ RocketMQGrpcSink<String> sink = RocketMQGrpcSink.<String>builder()
| `rocketmq.sink.lite-topic` | String | (none) | Lite topic within the main topic |
| `rocketmq.sink.max-attempts` | Integer | 3 | Producer send attempts |
-## SQL
+## 10. SQL
The SQL connector identifier is `rocketmq-grpc` (fat-jar module
`flink-sql-connector-rocketmq-grpc`). It exposes the same option keys as above, plus
-`rocketmq.source.fetch-concurrency`.
+`rocketmq.source.fetch-concurrency`. The SQL/Table path consumes message values only; the
+downstream ack / throttling APIs are DataStream-only.
+
+## 11. Known limitations
+
+- `LiteSimpleConsumer` is experimental in the SDK (5.2.1+) and offers synchronous APIs only.
+- One main lite topic per source; no startup-offset control in wildcard-subscription mode
+ (`OffsetOption` is limited to exact `subscribeLite`).
+- Throttling defers messages per message (extra RPC per defer, reordering, `deliveryAttempt`
+ growth); rely on the built-in safety valves and idempotent downstream processing.
+- The broker throttles dispatch for lite topics with large un-acked backlogs; design the
+ workload to spread across sub topics.
diff --git a/docs/sql-connector.md b/docs/sql-connector.md
index 9f0b015..4a65649 100644
--- a/docs/sql-connector.md
+++ b/docs/sql-connector.md
@@ -1,7 +1,9 @@
# Table / SQL Connector
The remoting SQL connector is packaged as the fat-jar `flink-sql-connector-rocketmq`
-(identifier `rocketmq`).
+(identifier `rocketmq`). The gRPC SQL connector is `flink-sql-connector-rocketmq-grpc`
+(identifier `rocketmq-grpc`); both fat-jars relocate their dependencies and can be deployed on
+the same classpath.
## Creating tables
diff --git a/flink-connector-rocketmq-grpc/pom.xml b/flink-connector-rocketmq-grpc/pom.xml
new file mode 100644
index 0000000..512ddce
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/pom.xml
@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-rocketmq-parent</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>flink-connector-rocketmq-grpc</artifactId>
+ <name>Flink : Connectors : RocketMQ gRPC</name>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <!-- Core -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Table ecosystem (optional, for Table API integration) -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-api-java-bridge</artifactId>
+ <scope>provided</scope>
+ <optional>true</optional>
+ </dependency>
+
+ <!-- RocketMQ gRPC client (5.x) -->
+
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client-java</artifactId>
+ </dependency>
+
+ <!-- Tests -->
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-streaming-java</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>analyze-deps</id>
+ <goals>
+ <goal>analyze</goal>
+ </goals>
+ <phase>verify</phase>
+ <configuration>
+ <ignoredUnusedDeclaredDependencies>
+ org.apache.flink:flink-table-api-java-bridge
+ </ignoredUnusedDeclaredDependencies>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcOptions.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcOptions.java
new file mode 100644
index 0000000..6948218
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcOptions.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+
+import java.time.Duration;
+
+/**
+ * Shared configuration options for the RocketMQ gRPC connector. These options describe how the
+ * underlying {@code rocketmq-client-java} SDK connects to the RocketMQ proxy (endpoints,
+ * credentials, TLS and namespace) and are consumed by both the source and the sink.
+ *
+ * <p>These are the programmatic/SDK-facing options used by the source and sink builders and their
+ * runtime; every key carries the {@link #CLIENT_CONFIG_PREFIX} prefix. The SQL DDL keys are defined
+ * separately in {@code RocketMQGrpcConnectorOptions}.
+ */
+@PublicEvolving
+public class RocketMQGrpcOptions {
+
+ private RocketMQGrpcOptions() {}
+
+ /** Prefix for the shared RocketMQ gRPC client options. */
+ public static final String CLIENT_CONFIG_PREFIX = "rocketmq.client.";
+
+ public static final ConfigOption<String> ENDPOINTS =
+ ConfigOptions.key(CLIENT_CONFIG_PREFIX + "endpoints")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The access point (proxy) endpoints the gRPC SDK communicates with, "
+ + "for example '127.0.0.1:8080'.");
+
+ public static final ConfigOption<String> NAMESPACE =
+ ConfigOptions.key(CLIENT_CONFIG_PREFIX + "namespace")
+ .stringType()
+ .defaultValue("")
+ .withDescription("The resource namespace of the RocketMQ instance.");
+
+ public static final ConfigOption<String> ACCESS_KEY =
+ ConfigOptions.key(CLIENT_CONFIG_PREFIX + "access-key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The access key used for static session credentials.");
+
+ public static final ConfigOption<String> SECRET_KEY =
+ ConfigOptions.key(CLIENT_CONFIG_PREFIX + "secret-key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The secret key used for static session credentials.");
+
+ /**
+ * The fully qualified class name of a {@link
+ * org.apache.flink.connector.rocketmq.grpc.common.CredentialsResolver} implementation (public,
+ * with a public no-argument constructor).
+ *
+ * <p>The resolver is instantiated reflectively on each TaskManager (never serialized) and
+ * resolves session credentials per proxy endpoint locally, e.g. from environment variables,
+ * mounted secret files or an external KMS. This keeps plaintext secrets out of the job graph
+ * and lets one downstream ack operator serve handles from multiple clusters with different
+ * credentials. When set, it takes precedence over {@link #ACCESS_KEY} / {@link #SECRET_KEY};
+ * returning {@code null} for an endpoint means no authentication is required.
+ */
+ public static final ConfigOption<String> CREDENTIALS_RESOLVER_CLASS =
+ ConfigOptions.key(CLIENT_CONFIG_PREFIX + "credentials-resolver-class")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The fully qualified class name of a CredentialsResolver that resolves "
+ + "session credentials per endpoint locally on the TaskManager "
+ + "(for example from environment variables, mounted secret files "
+ + "or an external KMS). When set, it takes precedence over the "
+ + "static access-key/secret-key options and keeps plaintext "
+ + "credentials out of the job graph.");
+
+ public static final ConfigOption<Boolean> TLS_ENABLED =
+ ConfigOptions.key(CLIENT_CONFIG_PREFIX + "tls-enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Whether TLS is enabled for the gRPC transport.");
+
+ public static final ConfigOption<Duration> REQUEST_TIMEOUT =
+ ConfigOptions.key(CLIENT_CONFIG_PREFIX + "request-timeout")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(3))
+ .withDescription("The request timeout for a single gRPC invocation.");
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessage.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessage.java
new file mode 100644
index 0000000..2ab5c4d
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessage.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import java.util.Objects;
+
+/**
+ * The element type produced by the RocketMQ gRPC source when running in downstream-acknowledgement
+ * mode. It pairs the deserialized record {@code value} with the credential-free {@link
+ * RocketMQReceiptHandle} needed to acknowledge (or re-schedule) the originating message from any
+ * downstream operator.
+ *
+ * @param <T> the deserialized record type.
+ */
+@PublicEvolving
+public final class AckableMessage<T> {
+
+ private final T value;
+ private final RocketMQReceiptHandle handle;
+
+ public AckableMessage(T value, RocketMQReceiptHandle handle) {
+ this.value = value;
+ this.handle = Objects.requireNonNull(handle, "handle should not be null");
+ }
+
+ /** The deserialized record value. */
+ public T getValue() {
+ return value;
+ }
+
+ /** The receipt handle used to acknowledge or re-schedule the originating message. */
+ public RocketMQReceiptHandle getHandle() {
+ return handle;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ AckableMessage<?> that = (AckableMessage<?>) o;
+ return Objects.equals(value, that.value) && handle.equals(that.handle);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(value, handle);
+ }
+
+ @Override
+ public String toString() {
+ return "AckableMessage{value=" + value + ", handle=" + handle + '}';
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageTypeInfo.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageTypeInfo.java
new file mode 100644
index 0000000..9e81050
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageTypeInfo.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * The {@link TypeInformation} for {@link AckableMessage}. It carries the value {@link
+ * TypeInformation} so that Flink can derive a serializer for {@code T} while the receipt handle is
+ * serialized by the dedicated {@link RocketMQReceiptHandle.Serializer}.
+ *
+ * @param <T> the deserialized record type.
+ */
+@PublicEvolving
+public final class AckableMessageTypeInfo<T> extends TypeInformation<AckableMessage<T>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final TypeInformation<T> valueTypeInfo;
+
+ public AckableMessageTypeInfo(TypeInformation<T> valueTypeInfo) {
+ this.valueTypeInfo = Objects.requireNonNull(valueTypeInfo, "valueTypeInfo");
+ }
+
+ @Override
+ public boolean isBasicType() {
+ return false;
+ }
+
+ @Override
+ public boolean isTupleType() {
+ return false;
+ }
+
+ @Override
+ public int getArity() {
+ return 2;
+ }
+
+ @Override
+ public int getTotalFields() {
+ return valueTypeInfo.getTotalFields() + 1;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Class<AckableMessage<T>> getTypeClass() {
+ return (Class<AckableMessage<T>>) (Class<?>) AckableMessage.class;
+ }
+
+ @Override
+ public boolean isKeyType() {
+ return false;
+ }
+
+ @Override
+ public TypeSerializer<AckableMessage<T>> createSerializer(ExecutionConfig config) {
+ return new Serializer<>(valueTypeInfo.createSerializer(config));
+ }
+
+ @Override
+ public String toString() {
+ return "AckableMessageTypeInfo<" + valueTypeInfo + '>';
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ AckableMessageTypeInfo<?> that = (AckableMessageTypeInfo<?>) obj;
+ return valueTypeInfo.equals(that.valueTypeInfo);
+ }
+
+ @Override
+ public int hashCode() {
+ return valueTypeInfo.hashCode();
+ }
+
+ @Override
+ public boolean canEqual(Object obj) {
+ return obj instanceof AckableMessageTypeInfo;
+ }
+
+ /**
+ * A {@link TypeSerializer} for {@link AckableMessage} that composes a caller-supplied value
+ * serializer with the {@link RocketMQReceiptHandle.Serializer}. A {@code null} value is
+ * supported via a boolean flag so that the value serializer never has to encode {@code null}.
+ *
+ * @param <T> the deserialized record type.
+ */
+ @Internal
+ public static final class Serializer<T> extends TypeSerializer<AckableMessage<T>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final TypeSerializer<T> valueSerializer;
+ private final TypeSerializer<RocketMQReceiptHandle> handleSerializer;
+
+ public Serializer(TypeSerializer<T> valueSerializer) {
+ this(valueSerializer, RocketMQReceiptHandle.Serializer.INSTANCE);
+ }
+
+ Serializer(
+ TypeSerializer<T> valueSerializer,
+ TypeSerializer<RocketMQReceiptHandle> handleSerializer) {
+ this.valueSerializer = valueSerializer;
+ this.handleSerializer = handleSerializer;
+ }
+
+ @Override
+ public boolean isImmutableType() {
+ return valueSerializer.isImmutableType();
+ }
+
+ @Override
+ public TypeSerializer<AckableMessage<T>> duplicate() {
+ final TypeSerializer<T> duplicatedValue = valueSerializer.duplicate();
+ if (duplicatedValue == valueSerializer) {
+ return this;
+ }
+ return new Serializer<>(duplicatedValue, handleSerializer.duplicate());
+ }
+
+ @Override
+ public AckableMessage<T> createInstance() {
+ return null;
+ }
+
+ @Override
+ public AckableMessage<T> copy(AckableMessage<T> from) {
+ final T value = from.getValue();
+ final T copiedValue = value == null ? null : valueSerializer.copy(value);
+ return new AckableMessage<>(copiedValue, from.getHandle());
+ }
+
+ @Override
+ public AckableMessage<T> copy(AckableMessage<T> from, AckableMessage<T> reuse) {
+ return copy(from);
+ }
+
+ @Override
+ public int getLength() {
+ return -1;
+ }
+
+ @Override
+ public void serialize(AckableMessage<T> record, DataOutputView target) throws IOException {
+ final T value = record.getValue();
+ if (value == null) {
+ target.writeBoolean(false);
+ } else {
+ target.writeBoolean(true);
+ valueSerializer.serialize(value, target);
+ }
+ handleSerializer.serialize(record.getHandle(), target);
+ }
+
+ @Override
+ public AckableMessage<T> deserialize(DataInputView source) throws IOException {
+ final T value = source.readBoolean() ? valueSerializer.deserialize(source) : null;
+ final RocketMQReceiptHandle handle = handleSerializer.deserialize(source);
+ return new AckableMessage<>(value, handle);
+ }
+
+ @Override
+ public AckableMessage<T> deserialize(AckableMessage<T> reuse, DataInputView source)
+ throws IOException {
+ return deserialize(source);
+ }
+
+ @Override
+ public void copy(DataInputView source, DataOutputView target) throws IOException {
+ serialize(deserialize(source), target);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ Serializer<?> that = (Serializer<?>) obj;
+ return valueSerializer.equals(that.valueSerializer)
+ && handleSerializer.equals(that.handleSerializer);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(valueSerializer, handleSerializer);
+ }
+
+ @Override
+ public TypeSerializerSnapshot<AckableMessage<T>> snapshotConfiguration() {
+ return new SerializerSnapshot<>(this);
+ }
+ }
+
+ /** Serializer snapshot that tracks the compatibility of the nested value serializer. */
+ public static final class SerializerSnapshot<T>
+ extends CompositeTypeSerializerSnapshot<AckableMessage<T>, Serializer<T>> {
+
+ private static final int CURRENT_VERSION = 1;
+
+ public SerializerSnapshot() {}
+
+ public SerializerSnapshot(Serializer<T> serializer) {
+ super(serializer);
+ }
+
+ @Override
+ protected int getCurrentOuterSnapshotVersion() {
+ return CURRENT_VERSION;
+ }
+
+ @Override
+ protected TypeSerializer<?>[] getNestedSerializers(Serializer<T> outerSerializer) {
+ return new TypeSerializer<?>[] {
+ outerSerializer.valueSerializer, outerSerializer.handleSerializer
+ };
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected Serializer<T> createOuterSerializerWithNestedSerializers(
+ TypeSerializer<?>[] nestedSerializers) {
+ return new Serializer<>(
+ (TypeSerializer<T>) nestedSerializers[0],
+ (TypeSerializer<RocketMQReceiptHandle>) nestedSerializers[1]);
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/MessageThrottlePolicy.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/MessageThrottlePolicy.java
new file mode 100644
index 0000000..1d45c6e
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/MessageThrottlePolicy.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.annotation.PublicEvolving;
+
+import java.io.Serializable;
+import java.time.Duration;
+import java.util.Optional;
+
+/**
+ * A per-message throttling policy used by {@link RocketMQThrottleProcessFunction}. For each value
+ * it decides whether the message should be acknowledged normally or deferred (throttled) by
+ * extending its invisible duration.
+ *
+ * @param <T> the value type carried by the {@code AckableMessage}.
+ */
+@PublicEvolving
+@FunctionalInterface
+public interface MessageThrottlePolicy<T> extends Serializable {
+
+ /**
+ * Decide how to handle the given value.
+ *
+ * @param value the deserialized message value.
+ * @return an empty {@link Optional} to acknowledge the message normally, or a positive {@link
+ * Duration} to defer its re-delivery by that amount (throttle).
+ */
+ Optional<Duration> onMessage(T value);
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQAckProcessFunction.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQAckProcessFunction.java
new file mode 100644
index 0000000..29330af
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQAckProcessFunction.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
+
+import java.time.Duration;
+import java.util.Objects;
+
+/**
+ * An abstract {@link ProcessFunction} that gives subclasses a ready-to-use {@link
+ * RocketMQLiteAckClient} so they can acknowledge or re-schedule RocketMQ Pop messages directly from
+ * their own business logic via {@link #ack} and {@link #changeInvisibleDuration}.
+ *
+ * <p>The client is shared per TaskManager through {@link RocketMQLiteAckClient#acquire}: it is
+ * acquired in {@link #open(OpenContext)} and released in {@link #close()}. The RocketMQ client
+ * options (endpoints, namespace, credentials, TLS, timeout) are provided through the {@link
+ * Configuration} passed to the constructor and never travel in the data stream.
+ *
+ * @param <IN> the input record type, typically {@code AckableMessage<T>}.
+ * @param <OUT> the output record type.
+ */
+@PublicEvolving
+public abstract class RocketMQAckProcessFunction<IN, OUT> extends ProcessFunction<IN, OUT> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Configuration configuration;
+
+ private transient RocketMQLiteAckClient ackClient;
+ private transient boolean acquired;
+ private transient Counter numAcksSucceeded;
+ private transient Counter numAcksFailed;
+ private transient Counter numInvisibleDurationChangesSucceeded;
+ private transient Counter numInvisibleDurationChangesFailed;
+
+ protected RocketMQAckProcessFunction(Configuration configuration) {
+ this.configuration =
+ new Configuration(
+ Objects.requireNonNull(configuration, "configuration should not be null"));
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ super.open(openContext);
+ this.ackClient = RocketMQLiteAckClient.acquire(configuration);
+ this.acquired = true;
+ this.numAcksSucceeded = getRuntimeContext().getMetricGroup().counter("numAcksSucceeded");
+ this.numAcksFailed = getRuntimeContext().getMetricGroup().counter("numAcksFailed");
+ this.numInvisibleDurationChangesSucceeded =
+ getRuntimeContext()
+ .getMetricGroup()
+ .counter("numInvisibleDurationChangesSucceeded");
+ this.numInvisibleDurationChangesFailed =
+ getRuntimeContext().getMetricGroup().counter("numInvisibleDurationChangesFailed");
+ }
+
+ @Override
+ public void close() throws Exception {
+ // Only release when open() actually acquired the client; Flink calls close() even after a
+ // failed open(), and a stray release would decrement another operator's reference.
+ if (acquired) {
+ RocketMQLiteAckClient.release(configuration);
+ acquired = false;
+ }
+ this.ackClient = null;
+ super.close();
+ }
+
+ /** Acknowledge the message described by the handle so it is not re-delivered. */
+ protected void ack(RocketMQReceiptHandle handle) {
+ try {
+ ackClient.ack(handle);
+ numAcksSucceeded.inc();
+ } catch (RuntimeException e) {
+ numAcksFailed.inc();
+ throw e;
+ }
+ }
+
+ /**
+ * Change the invisible duration of the message described by the handle, deferring its next
+ * re-delivery by {@code invisibleDuration}.
+ */
+ protected void changeInvisibleDuration(
+ RocketMQReceiptHandle handle, Duration invisibleDuration) {
+ try {
+ ackClient.changeInvisibleDuration(handle, invisibleDuration);
+ numInvisibleDurationChangesSucceeded.inc();
+ } catch (RuntimeException e) {
+ numInvisibleDurationChangesFailed.inc();
+ throw e;
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClient.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClient.java
new file mode 100644
index 0000000..f6754c4
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClient.java
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+import org.apache.flink.connector.rocketmq.grpc.common.ClientConfigurationProvider;
+import org.apache.flink.connector.rocketmq.grpc.common.CredentialsResolver;
+import org.apache.flink.connector.rocketmq.grpc.common.CredentialsResolvers;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.StringUtils;
+
+import org.apache.rocketmq.client.apis.ClientConfiguration;
+import org.apache.rocketmq.client.apis.ClientException;
+import org.apache.rocketmq.client.apis.ClientServiceProvider;
+import org.apache.rocketmq.client.apis.consumer.LiteSimpleConsumer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A credential-free acknowledgement client that a downstream operator uses to acknowledge or
+ * re-schedule RocketMQ Pop messages described by a {@link RocketMQReceiptHandle}. The credentials
+ * used to talk to the RocketMQ proxy are configured on the operator that owns the client and are
+ * <b>never</b> carried in the data stream.
+ *
+ * <p>The client keeps a lazily populated pool of same-group {@link LiteSimpleConsumer}s, one per
+ * {@link ConsumerKey routing triple} carried by the incoming handles, and issues the {@code ack} /
+ * {@code changeInvisibleDuration} RPC through the consumer that matches the handle. The pooled
+ * consumer for a handle is built with a {@link ClientConfiguration} whose endpoints and namespace
+ * come from the handle (so the ack RPC is routed to the right proxy and resource) and whose
+ * credentials/TLS/timeout come from the operator {@link Configuration}. This is required because
+ * the SDK stamps the ack request with the <em>consumer's</em> namespace, so a pooled consumer must
+ * share the handle's namespace.
+ *
+ * <p>Instances are shared per TaskManager JVM: {@link #acquire(Configuration)} reference-counts a
+ * client per distinct client configuration and {@link #release(Configuration)} closes it once no
+ * operator instance references it anymore, so callers must not close a client directly.
+ */
+@Internal
+public final class RocketMQLiteAckClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RocketMQLiteAckClient.class);
+
+ /**
+ * The await duration is only consulted by {@code receive}, which this ack-only client never
+ * calls; a small non-null value is enough to satisfy the builder.
+ */
+ private static final Duration ACK_ONLY_AWAIT_DURATION = Duration.ofSeconds(5);
+
+ private static final int MAX_RPC_ATTEMPTS = 3;
+ private static final long RPC_RETRY_BACKOFF_INITIAL_MS = 100L;
+
+ /** {@code apache.rocketmq.v2.Code.INVALID_RECEIPT_HANDLE}, embedded in exception messages. */
+ private static final String INVALID_RECEIPT_HANDLE_MARKER = "response-code=40013";
+
+ private static final Map<String, RefCounted> CLIENTS = new HashMap<>();
+
+ private final Configuration configuration;
+ @Nullable private final CredentialsResolver credentialsResolver;
+ private final Map<ConsumerKey, LiteSimpleConsumer> consumers = new ConcurrentHashMap<>();
+
+ private volatile boolean closed = false;
+
+ private RocketMQLiteAckClient(Configuration configuration) {
+ this.configuration =
+ Objects.requireNonNull(configuration, "configuration should not be null");
+ this.credentialsResolver = CredentialsResolvers.createFromConfiguration(configuration);
+ }
+
+ /**
+ * Acquire the shared ack client for the given client configuration, creating it if necessary
+ * and incrementing its reference count.
+ *
+ * @param configuration the operator configuration carrying the RocketMQ client options.
+ * @return the shared ack client; callers must not close it directly.
+ */
+ public static synchronized RocketMQLiteAckClient acquire(Configuration configuration) {
+ Objects.requireNonNull(configuration, "configuration should not be null");
+ final String key = keyOf(configuration);
+ RefCounted ref = CLIENTS.get(key);
+ if (ref == null) {
+ ref = new RefCounted(new RocketMQLiteAckClient(configuration));
+ CLIENTS.put(key, ref);
+ }
+ ref.count++;
+ return ref.client;
+ }
+
+ /**
+ * Release a previously {@link #acquire(Configuration) acquired} client, decrementing its
+ * reference count and closing it once no operator instance references it anymore.
+ *
+ * <p>The client is closed outside the registry lock so that a slow close (it issues network
+ * calls) cannot block concurrent {@code acquire}/{@code release} calls of other operators.
+ *
+ * @param configuration the same configuration that was passed to {@link
+ * #acquire(Configuration)}.
+ */
+ public static void release(Configuration configuration) {
+ Objects.requireNonNull(configuration, "configuration should not be null");
+ final String key = keyOf(configuration);
+ RocketMQLiteAckClient toClose = null;
+ synchronized (RocketMQLiteAckClient.class) {
+ final RefCounted ref = CLIENTS.get(key);
+ if (ref == null) {
+ return;
+ }
+ ref.count--;
+ if (ref.count <= 0) {
+ CLIENTS.remove(key);
+ toClose = ref.client;
+ }
+ }
+ if (toClose != null) {
+ toClose.close();
+ }
+ }
+
+ @VisibleForTesting
+ static synchronized int getReferenceCount(Configuration configuration) {
+ final RefCounted ref = CLIENTS.get(keyOf(configuration));
+ return ref == null ? 0 : ref.count;
+ }
+
+ /**
+ * Build a stable pool key from the credential-bearing client options. Distinct credentials,
+ * endpoints, namespaces, TLS settings or timeouts each get their own shared client. The secret
+ * key is hashed so the plaintext secret is never held in the registry key.
+ */
+ private static String keyOf(Configuration configuration) {
+ return configuration.get(RocketMQGrpcOptions.ENDPOINTS)
+ + '|'
+ + configuration.get(RocketMQGrpcOptions.NAMESPACE)
+ + '|'
+ + configuration.get(RocketMQGrpcOptions.ACCESS_KEY)
+ + '|'
+ + sha256(configuration.get(RocketMQGrpcOptions.SECRET_KEY))
+ + '|'
+ + configuration.get(RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS)
+ + '|'
+ + configuration.get(RocketMQGrpcOptions.TLS_ENABLED)
+ + '|'
+ + configuration.get(RocketMQGrpcOptions.REQUEST_TIMEOUT);
+ }
+
+ private static String sha256(@Nullable String value) {
+ if (value == null) {
+ return "";
+ }
+ try {
+ final MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ return StringUtils.byteToHexString(
+ digest.digest(value.getBytes(StandardCharsets.UTF_8)));
+ } catch (NoSuchAlgorithmException e) {
+ throw new FlinkRuntimeException("SHA-256 is not available.", e);
+ }
+ }
+
+ /**
+ * Acknowledge the message described by the handle, removing it from the Pop invisible set so it
+ * is not re-delivered.
+ *
+ * @param handle the credential-free receipt handle of the message to acknowledge.
+ */
+ public void ack(RocketMQReceiptHandle handle) {
+ Objects.requireNonNull(handle, "handle should not be null");
+ final LiteSimpleConsumer consumer = getOrCreateConsumer(handle);
+ invokeWithRetry(
+ "acknowledge",
+ handle,
+ () -> consumer.ack(RocketMQReceiptHandleCodec.toAckable(handle)));
+ }
+
+ /**
+ * Change the invisible duration of the message described by the handle. Extending the invisible
+ * duration defers the next possible re-delivery of the message, which downstream operators use
+ * to throttle a (sub) topic without dropping the message.
+ *
+ * @param handle the credential-free receipt handle of the message.
+ * @param invisibleDuration the new invisible duration counted from now.
+ */
+ public void changeInvisibleDuration(RocketMQReceiptHandle handle, Duration invisibleDuration) {
+ Objects.requireNonNull(handle, "handle should not be null");
+ Objects.requireNonNull(invisibleDuration, "invisibleDuration should not be null");
+ final LiteSimpleConsumer consumer = getOrCreateConsumer(handle);
+ invokeWithRetry(
+ "change the invisible duration of",
+ handle,
+ () ->
+ consumer.changeInvisibleDuration(
+ RocketMQReceiptHandleCodec.toAckable(handle), invisibleDuration));
+ }
+
+ /**
+ * Run the RPC with a bounded exponential-backoff retry. An expired/invalid receipt handle is
+ * tolerated: the message will simply be redelivered by the broker (at-least-once), so failing
+ * the job for it would be worse than the duplicate.
+ */
+ private void invokeWithRetry(String action, RocketMQReceiptHandle handle, AckRpc rpc) {
+ long backoffMs = RPC_RETRY_BACKOFF_INITIAL_MS;
+ for (int attempt = 1; ; attempt++) {
+ try {
+ rpc.run();
+ return;
+ } catch (ClientException e) {
+ if (isInvalidReceiptHandle(e)) {
+ LOG.warn(
+ "The receipt handle of message {} is invalid or expired; the message "
+ + "will be redelivered by the broker.",
+ handle.getMessageId(),
+ e);
+ return;
+ }
+ if (attempt >= MAX_RPC_ATTEMPTS) {
+ throw new FlinkRuntimeException(
+ "Failed to "
+ + action
+ + " RocketMQ message "
+ + handle.getMessageId()
+ + " after "
+ + attempt
+ + " attempts",
+ e);
+ }
+ LOG.warn(
+ "Failed to {} RocketMQ message {} (attempt {}/{}), retrying in {} ms",
+ action,
+ handle.getMessageId(),
+ attempt,
+ MAX_RPC_ATTEMPTS,
+ backoffMs,
+ e);
+ try {
+ Thread.sleep(backoffMs);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ final FlinkRuntimeException interrupted =
+ new FlinkRuntimeException(
+ "Interrupted while retrying to "
+ + action
+ + " RocketMQ message "
+ + handle.getMessageId(),
+ ie);
+ interrupted.addSuppressed(e);
+ throw interrupted;
+ }
+ backoffMs *= 2;
+ }
+ }
+ }
+
+ private static boolean isInvalidReceiptHandle(ClientException e) {
+ for (Throwable t = e; t != null; t = t.getCause()) {
+ final String message = t.getMessage();
+ if (message != null && message.contains(INVALID_RECEIPT_HANDLE_MARKER)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** An ack-related RPC over a pooled consumer. */
+ @FunctionalInterface
+ private interface AckRpc {
+ void run() throws ClientException;
+ }
+
+ /** Close all pooled consumers. Invoked by {@link #release(Configuration)} on last release. */
+ private void close() {
+ closed = true;
+ for (Map.Entry<ConsumerKey, LiteSimpleConsumer> entry : consumers.entrySet()) {
+ try {
+ entry.getValue().close();
+ } catch (Exception e) {
+ LOG.warn("Failed to close the ack consumer for {}.", entry.getKey(), e);
+ }
+ }
+ consumers.clear();
+ }
+
+ private LiteSimpleConsumer getOrCreateConsumer(RocketMQReceiptHandle handle) {
+ if (closed) {
+ throw new IllegalStateException("The ack client has been closed.");
+ }
+ final ConsumerKey key =
+ new ConsumerKey(
+ handle.getEndpoint(), handle.getNamespace(), handle.getConsumerGroup());
+ return consumers.computeIfAbsent(key, k -> createConsumer(k, handle.getTopic()));
+ }
+
+ private LiteSimpleConsumer createConsumer(ConsumerKey key, String bindTopic) {
+ try {
+ final ClientServiceProvider provider = ClientServiceProvider.loadService();
+ return provider.newLiteSimpleConsumerBuilder()
+ .setClientConfiguration(
+ ClientConfigurationProvider.getClientConfiguration(
+ configuration,
+ key.endpoint,
+ key.namespace,
+ credentialsResolver))
+ .setConsumerGroup(key.consumerGroup)
+ .setAwaitDuration(ACK_ONLY_AWAIT_DURATION)
+ .bindTopic(bindTopic)
+ .build();
+ } catch (ClientException e) {
+ throw new FlinkRuntimeException("Failed to create the ack consumer for " + key, e);
+ }
+ }
+
+ private static final class RefCounted {
+ private final RocketMQLiteAckClient client;
+ private int count;
+
+ private RefCounted(RocketMQLiteAckClient client) {
+ this.client = client;
+ }
+ }
+
+ /**
+ * The identity of a pooled ack consumer: a message can only be acknowledged by a consumer in
+ * the same group, pointing at the same proxy endpoint(s) and resource namespace.
+ */
+ private static final class ConsumerKey {
+
+ private final String endpoint;
+ private final String namespace;
+ private final String consumerGroup;
+
+ private ConsumerKey(String endpoint, String namespace, String consumerGroup) {
+ this.endpoint = endpoint;
+ this.namespace = namespace;
+ this.consumerGroup = consumerGroup;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ConsumerKey that = (ConsumerKey) o;
+ return endpoint.equals(that.endpoint)
+ && namespace.equals(that.namespace)
+ && consumerGroup.equals(that.consumerGroup);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(endpoint, namespace, consumerGroup);
+ }
+
+ @Override
+ public String toString() {
+ return "ConsumerKey{"
+ + "endpoint='"
+ + endpoint
+ + '\''
+ + ", namespace='"
+ + namespace
+ + '\''
+ + ", consumerGroup='"
+ + consumerGroup
+ + '\''
+ + '}';
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandle.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandle.java
new file mode 100644
index 0000000..eb8c43c
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandle.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.typeutils.SimpleTypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * A self-contained, serializable descriptor of a RocketMQ Pop message that carries everything a
+ * downstream operator needs to acknowledge (or extend the invisible duration of) the message
+ * without holding on to the SDK message object or any credentials.
+ *
+ * <p>The RocketMQ 5.x Pop model lets any consumer in the same group acknowledge a message given its
+ * receipt handle and routing information; the message does not have to be acked by the consumer
+ * that originally received it. This class captures exactly that routing information so that the
+ * acknowledgement decision can be moved to any downstream operator:
+ *
+ * <ul>
+ * <li>{@code endpoint}, {@code namespace}, {@code consumerGroup} — the routing triple that
+ * selects which pooled consumer must issue the ack RPC. Multiple sources (possibly pointing
+ * at different clusters/namespaces/groups) can coexist because each message carries its own
+ * triple.
+ * <li>{@code topic}, {@code liteTopic}, {@code messageId}, {@code receiptHandle}, {@code
+ * deliveryAttempt} — the per-message fields required to rebuild the SDK message view used by
+ * the ack/changeInvisibleDuration calls.
+ * </ul>
+ *
+ * <p>This type intentionally contains <b>no</b> credentials, no protobuf blob and no message body,
+ * so it is safe to ship across the Flink data stream.
+ */
+@PublicEvolving
+public final class RocketMQReceiptHandle implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String endpoint;
+ private final String namespace;
+ private final String consumerGroup;
+ private final String topic;
+ /** The lite (sub) topic, or {@code null} when the message was not received via a lite topic. */
+ private final String liteTopic;
+
+ private final String messageId;
+ private final String receiptHandle;
+ private final int deliveryAttempt;
+
+ public RocketMQReceiptHandle(
+ String endpoint,
+ String namespace,
+ String consumerGroup,
+ String topic,
+ String liteTopic,
+ String messageId,
+ String receiptHandle,
+ int deliveryAttempt) {
+ this.endpoint = Objects.requireNonNull(endpoint, "endpoint should not be null");
+ this.namespace = Objects.requireNonNull(namespace, "namespace should not be null");
+ this.consumerGroup =
+ Objects.requireNonNull(consumerGroup, "consumerGroup should not be null");
+ this.topic = Objects.requireNonNull(topic, "topic should not be null");
+ this.liteTopic = liteTopic;
+ this.messageId = Objects.requireNonNull(messageId, "messageId should not be null");
+ this.receiptHandle =
+ Objects.requireNonNull(receiptHandle, "receiptHandle should not be null");
+ this.deliveryAttempt = deliveryAttempt;
+ }
+
+ /**
+ * The proxy endpoint(s) the message came from, as a {@code host:port[;host:port...]} string
+ * that can be parsed back into the SDK endpoints.
+ */
+ public String getEndpoint() {
+ return endpoint;
+ }
+
+ /** The resource namespace of the RocketMQ instance ({@code ""} when none is configured). */
+ public String getNamespace() {
+ return namespace;
+ }
+
+ /**
+ * The consumer group the source used; the ack must be issued by a consumer of the same group.
+ */
+ public String getConsumerGroup() {
+ return consumerGroup;
+ }
+
+ /** The (physical) topic the message belongs to. */
+ public String getTopic() {
+ return topic;
+ }
+
+ /** The lite (sub) topic, or {@code null} when the message was not received via a lite topic. */
+ public String getLiteTopic() {
+ return liteTopic;
+ }
+
+ /** The unique message id. */
+ public String getMessageId() {
+ return messageId;
+ }
+
+ /** The Pop receipt handle used to acknowledge or re-schedule the message. */
+ public String getReceiptHandle() {
+ return receiptHandle;
+ }
+
+ /** The number of times the message has been delivered. */
+ public int getDeliveryAttempt() {
+ return deliveryAttempt;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ RocketMQReceiptHandle that = (RocketMQReceiptHandle) o;
+ return deliveryAttempt == that.deliveryAttempt
+ && endpoint.equals(that.endpoint)
+ && namespace.equals(that.namespace)
+ && consumerGroup.equals(that.consumerGroup)
+ && topic.equals(that.topic)
+ && Objects.equals(liteTopic, that.liteTopic)
+ && messageId.equals(that.messageId)
+ && receiptHandle.equals(that.receiptHandle);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ endpoint,
+ namespace,
+ consumerGroup,
+ topic,
+ liteTopic,
+ messageId,
+ receiptHandle,
+ deliveryAttempt);
+ }
+
+ @Override
+ public String toString() {
+ return "RocketMQReceiptHandle{"
+ + "endpoint='"
+ + endpoint
+ + '\''
+ + ", namespace='"
+ + namespace
+ + '\''
+ + ", consumerGroup='"
+ + consumerGroup
+ + '\''
+ + ", topic='"
+ + topic
+ + '\''
+ + ", liteTopic='"
+ + liteTopic
+ + '\''
+ + ", messageId='"
+ + messageId
+ + '\''
+ + ", deliveryAttempt="
+ + deliveryAttempt
+ + '}';
+ }
+
+ /**
+ * A stateless {@link org.apache.flink.api.common.typeutils.TypeSerializer} for {@link
+ * RocketMQReceiptHandle}. It writes the eight self-contained fields directly; the nullable
+ * {@code liteTopic} is guarded by a boolean flag.
+ */
+ @Internal
+ public static final class Serializer extends TypeSerializerSingleton<RocketMQReceiptHandle> {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final Serializer INSTANCE = new Serializer();
+
+ @Override
+ public boolean isImmutableType() {
+ return true;
+ }
+
+ @Override
+ public RocketMQReceiptHandle createInstance() {
+ return null;
+ }
+
+ @Override
+ public RocketMQReceiptHandle copy(RocketMQReceiptHandle from) {
+ // RocketMQReceiptHandle is immutable.
+ return from;
+ }
+
+ @Override
+ public RocketMQReceiptHandle copy(RocketMQReceiptHandle from, RocketMQReceiptHandle reuse) {
+ return from;
+ }
+
+ @Override
+ public int getLength() {
+ return -1;
+ }
+
+ @Override
+ public void serialize(RocketMQReceiptHandle record, DataOutputView target)
+ throws IOException {
+ target.writeUTF(record.getEndpoint());
+ target.writeUTF(record.getNamespace());
+ target.writeUTF(record.getConsumerGroup());
+ target.writeUTF(record.getTopic());
+ final String liteTopic = record.getLiteTopic();
+ if (liteTopic == null) {
+ target.writeBoolean(false);
+ } else {
+ target.writeBoolean(true);
+ target.writeUTF(liteTopic);
+ }
+ target.writeUTF(record.getMessageId());
+ target.writeUTF(record.getReceiptHandle());
+ target.writeInt(record.getDeliveryAttempt());
+ }
+
+ @Override
+ public RocketMQReceiptHandle deserialize(DataInputView source) throws IOException {
+ final String endpoint = source.readUTF();
+ final String namespace = source.readUTF();
+ final String consumerGroup = source.readUTF();
+ final String topic = source.readUTF();
+ final String liteTopic = source.readBoolean() ? source.readUTF() : null;
+ final String messageId = source.readUTF();
+ final String receiptHandle = source.readUTF();
+ final int deliveryAttempt = source.readInt();
+ return new RocketMQReceiptHandle(
+ endpoint,
+ namespace,
+ consumerGroup,
+ topic,
+ liteTopic,
+ messageId,
+ receiptHandle,
+ deliveryAttempt);
+ }
+
+ @Override
+ public RocketMQReceiptHandle deserialize(RocketMQReceiptHandle reuse, DataInputView source)
+ throws IOException {
+ return deserialize(source);
+ }
+
+ @Override
+ public void copy(DataInputView source, DataOutputView target) throws IOException {
+ serialize(deserialize(source), target);
+ }
+
+ @Override
+ public TypeSerializerSnapshot<RocketMQReceiptHandle> snapshotConfiguration() {
+ return new SerializerSnapshot();
+ }
+
+ /** Serializer snapshot for {@link Serializer}. */
+ public static final class SerializerSnapshot
+ extends SimpleTypeSerializerSnapshot<RocketMQReceiptHandle> {
+
+ public SerializerSnapshot() {
+ super(() -> INSTANCE);
+ }
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodec.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodec.java
new file mode 100644
index 0000000..8f15bf2
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodec.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.annotation.Internal;
+
+import org.apache.rocketmq.client.apis.message.MessageId;
+import org.apache.rocketmq.client.apis.message.MessageView;
+import org.apache.rocketmq.client.java.message.MessageIdCodec;
+import org.apache.rocketmq.client.java.message.MessageViewImpl;
+import org.apache.rocketmq.client.java.route.Address;
+import org.apache.rocketmq.client.java.route.Endpoints;
+import org.apache.rocketmq.client.java.route.MessageQueueImpl;
+
+import java.util.Collections;
+
+/**
+ * The single adapter that bridges the connector's credential-free {@link RocketMQReceiptHandle} and
+ * the {@code rocketmq-client-java} internal implementation classes. It is deliberately the
+ * <b>only</b> class in the connector that depends on the SDK {@code
+ * org.apache.rocketmq.client.java.*} internals, so that the (experimental) coupling is confined to
+ * one place.
+ *
+ * <p>{@link #extract} runs on the source side and pulls the routing/ack fields out of an SDK
+ * message view. {@link #toAckable} runs on the downstream ack side and rebuilds the minimal SDK
+ * message view that {@code LiteSimpleConsumer#ack} / {@code changeInvisibleDuration} require (both
+ * perform an {@code instanceof MessageViewImpl} downcast internally and only read the topic,
+ * message id, receipt handle, lite topic and endpoints).
+ */
+@Internal
+public final class RocketMQReceiptHandleCodec {
+
+ private RocketMQReceiptHandleCodec() {}
+
+ /**
+ * Extract a credential-free receipt handle from an SDK message view.
+ *
+ * @param view the SDK message view returned by the consumer.
+ * @param namespace the resource namespace configured on the source.
+ * @param consumerGroup the consumer group configured on the source.
+ * @return a self-contained {@link RocketMQReceiptHandle}.
+ * @throws IllegalStateException if the view is not the expected internal implementation type.
+ */
+ public static RocketMQReceiptHandle extract(
+ MessageView view, String namespace, String consumerGroup) {
+ if (!(view instanceof MessageViewImpl)) {
+ throw new IllegalStateException(
+ "Expected a "
+ + MessageViewImpl.class.getName()
+ + " but got "
+ + (view == null ? "null" : view.getClass().getName())
+ + "; the RocketMQ SDK internal message type has changed.");
+ }
+ final MessageViewImpl impl = (MessageViewImpl) view;
+ final Endpoints endpoints = impl.getEndpoints();
+ if (endpoints == null) {
+ throw new IllegalStateException(
+ "The message view does not carry endpoints, cannot build a receipt handle for "
+ + impl.getMessageId());
+ }
+ return new RocketMQReceiptHandle(
+ toEndpointString(endpoints),
+ namespace,
+ consumerGroup,
+ impl.getTopic(),
+ impl.getLiteTopic().orElse(null),
+ impl.getMessageId().toString(),
+ impl.getReceiptHandle(),
+ impl.getDeliveryAttempt());
+ }
+
+ /**
+ * Render the endpoints as a {@code host:port[;host:port...]} string that round-trips through
+ * {@code new Endpoints(String)}. The {@code Endpoints#getFacade()} form is not used because it
+ * carries a scheme prefix ({@code ipv4:}/{@code ipv6:}/{@code dns:}) that the string
+ * constructor cannot parse back.
+ */
+ private static String toEndpointString(Endpoints endpoints) {
+ final StringBuilder builder = new StringBuilder();
+ for (Address address : endpoints.getAddresses()) {
+ if (builder.length() > 0) {
+ builder.append(';');
+ }
+ builder.append(address.getAddress());
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Rebuild the minimal SDK message view required to acknowledge or re-schedule the message
+ * described by the given handle. The reconstructed view carries an empty body and no user
+ * properties; only the fields consulted by the ack / changeInvisibleDuration RPC are populated.
+ *
+ * @param handle the credential-free receipt handle.
+ * @return an SDK {@link MessageView} suitable for {@code ack} / {@code
+ * changeInvisibleDuration}.
+ */
+ public static MessageView toAckable(RocketMQReceiptHandle handle) {
+ final MessageId messageId = MessageIdCodec.getInstance().decode(handle.getMessageId());
+ final MessageQueueImpl messageQueue = buildMessageQueue(handle);
+ return new MessageViewImpl(
+ messageId,
+ handle.getTopic(),
+ new byte[0],
+ null,
+ null,
+ handle.getLiteTopic(),
+ null,
+ null,
+ Collections.emptyList(),
+ Collections.emptyMap(),
+ "",
+ 0L,
+ handle.getDeliveryAttempt(),
+ messageQueue,
+ handle.getReceiptHandle(),
+ 0L,
+ false,
+ null);
+ }
+
+ private static MessageQueueImpl buildMessageQueue(RocketMQReceiptHandle handle) {
+ final apache.rocketmq.v2.Endpoints endpoints =
+ new Endpoints(handle.getEndpoint()).toProtobuf();
+ final apache.rocketmq.v2.Broker broker =
+ apache.rocketmq.v2.Broker.newBuilder()
+ .setName("")
+ .setId(0)
+ .setEndpoints(endpoints)
+ .build();
+ final apache.rocketmq.v2.Resource topic =
+ apache.rocketmq.v2.Resource.newBuilder()
+ .setResourceNamespace(handle.getNamespace())
+ .setName(handle.getTopic())
+ .build();
+ // A concrete permission is required: MessageQueueImpl rejects PERMISSION_UNSPECIFIED.
+ final apache.rocketmq.v2.MessageQueue messageQueue =
+ apache.rocketmq.v2.MessageQueue.newBuilder()
+ .setTopic(topic)
+ .setId(0)
+ .setPermission(apache.rocketmq.v2.Permission.READ)
+ .setBroker(broker)
+ .build();
+ return new MessageQueueImpl(messageQueue);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunction.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunction.java
new file mode 100644
index 0000000..0f0891b
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunction.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.functions.ProcessFunction;
+import org.apache.flink.util.Collector;
+
+import java.time.Duration;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * A convenience one-stop throttle operator. For every {@link AckableMessage} it consults a {@link
+ * MessageThrottlePolicy}: when the policy returns a delay it {@linkplain
+ * RocketMQAckProcessFunction#changeInvisibleDuration changes the invisible duration} (deferring the
+ * message to throttle its (sub) topic); otherwise it {@linkplain RocketMQAckProcessFunction#ack
+ * acknowledges} the message. The underlying value is always forwarded downstream unchanged.
+ *
+ * <p>Two safeguards prevent a persistently throttled message from being deferred forever and
+ * eventually pushed into the dead-letter queue once its delivery attempts are exhausted:
+ *
+ * <ul>
+ * <li>{@code maxDeliveryAttempt} — once the message's delivery attempt reaches this bound the
+ * message is acknowledged instead of deferred again.
+ * <li>{@code maxInvisibleDuration} — every requested delay is capped to this value.
+ * </ul>
+ *
+ * @param <T> the value type carried by the {@code AckableMessage}.
+ */
+@PublicEvolving
+public class RocketMQThrottleProcessFunction<T>
+ extends RocketMQAckProcessFunction<AckableMessage<T>, T> {
+
+ private static final long serialVersionUID = 1L;
+
+ /** Default cap on the number of times a message may be deferred before it is acknowledged. */
+ public static final int DEFAULT_MAX_DELIVERY_ATTEMPT = 16;
+
+ /** Default cap on a single requested invisible duration. */
+ public static final Duration DEFAULT_MAX_INVISIBLE_DURATION = Duration.ofMinutes(30);
+
+ private final MessageThrottlePolicy<T> policy;
+ private final int maxDeliveryAttempt;
+ private final Duration maxInvisibleDuration;
+
+ public RocketMQThrottleProcessFunction(
+ Configuration configuration, MessageThrottlePolicy<T> policy) {
+ this(configuration, policy, DEFAULT_MAX_DELIVERY_ATTEMPT, DEFAULT_MAX_INVISIBLE_DURATION);
+ }
+
+ public RocketMQThrottleProcessFunction(
+ Configuration configuration,
+ MessageThrottlePolicy<T> policy,
+ int maxDeliveryAttempt,
+ Duration maxInvisibleDuration) {
+ super(configuration);
+ this.policy = Objects.requireNonNull(policy, "policy should not be null");
+ this.maxDeliveryAttempt = maxDeliveryAttempt;
+ this.maxInvisibleDuration =
+ Objects.requireNonNull(
+ maxInvisibleDuration, "maxInvisibleDuration should not be null");
+ }
+
+ @Override
+ public void processElement(
+ AckableMessage<T> message,
+ ProcessFunction<AckableMessage<T>, T>.Context context,
+ Collector<T> out) {
+ final RocketMQReceiptHandle handle = message.getHandle();
+ final Optional<Duration> delay = policy.onMessage(message.getValue());
+
+ if (delay.isPresent() && handle.getDeliveryAttempt() < maxDeliveryAttempt) {
+ changeInvisibleDuration(handle, capDelay(delay.get()));
+ } else {
+ // Either the policy accepted the message, or the defer safeguard tripped: ack it so it
+ // is not eventually pushed into the dead-letter queue.
+ ack(handle);
+ }
+
+ out.collect(message.getValue());
+ }
+
+ private Duration capDelay(Duration requested) {
+ return requested.compareTo(maxInvisibleDuration) > 0 ? maxInvisibleDuration : requested;
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/ClientConfigurationProvider.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/ClientConfigurationProvider.java
new file mode 100644
index 0000000..d0f9bb6
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/ClientConfigurationProvider.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.common;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+import org.apache.flink.util.StringUtils;
+
+import org.apache.rocketmq.client.apis.ClientConfiguration;
+import org.apache.rocketmq.client.apis.ClientConfigurationBuilder;
+
+import javax.annotation.Nullable;
+
+/**
+ * Builds an SDK {@link ClientConfiguration} from a Flink {@link Configuration}. The produced
+ * configuration is shared by the source ({@code SimpleConsumer}) and the sink ({@code Producer}).
+ */
+@Internal
+public class ClientConfigurationProvider {
+
+ private ClientConfigurationProvider() {}
+
+ /** Build a {@link ClientConfiguration} from the given Flink configuration. */
+ public static ClientConfiguration getClientConfiguration(Configuration configuration) {
+ final String endpoints = configuration.get(RocketMQGrpcOptions.ENDPOINTS);
+ if (StringUtils.isNullOrWhitespaceOnly(endpoints)) {
+ throw new IllegalArgumentException(
+ "The endpoints of the RocketMQ gRPC connector must be configured.");
+ }
+ return getClientConfiguration(
+ configuration,
+ endpoints,
+ configuration.get(RocketMQGrpcOptions.NAMESPACE),
+ CredentialsResolvers.createFromConfiguration(configuration));
+ }
+
+ /**
+ * Build a {@link ClientConfiguration} whose endpoints and namespace are supplied by the caller
+ * (e.g. taken from a receipt handle), while the credentials, TLS and request timeout come from
+ * the Flink configuration.
+ */
+ public static ClientConfiguration getClientConfiguration(
+ Configuration configuration,
+ String endpoints,
+ @Nullable String namespace,
+ @Nullable CredentialsResolver credentialsResolver) {
+ final ClientConfigurationBuilder builder =
+ ClientConfiguration.newBuilder()
+ .setEndpoints(endpoints)
+ .setRequestTimeout(configuration.get(RocketMQGrpcOptions.REQUEST_TIMEOUT))
+ .enableSsl(configuration.get(RocketMQGrpcOptions.TLS_ENABLED));
+
+ if (!StringUtils.isNullOrWhitespaceOnly(namespace)) {
+ builder.setNamespace(namespace);
+ }
+
+ CredentialsResolvers.applyCredentials(
+ builder, endpoints, configuration, credentialsResolver);
+
+ return builder.build();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolver.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolver.java
new file mode 100644
index 0000000..1099820
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolver.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.common;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.Configuration;
+
+import org.apache.rocketmq.client.apis.SessionCredentialsProvider;
+
+import javax.annotation.Nullable;
+
+/**
+ * Resolves RocketMQ session credentials for a given proxy endpoint on the TaskManager, so that
+ * plaintext access/secret keys never need to be placed in the job graph or the Flink {@link
+ * Configuration}.
+ *
+ * <p>Implementations are instantiated reflectively from the class name configured under {@code
+ * rocketmq.client.credentials-resolver-class} and therefore must be public and provide a public
+ * no-argument constructor. The instance is created locally on each TaskManager (never serialized),
+ * which makes it safe to read credentials from environment variables, mounted secret files or an
+ * external KMS.
+ *
+ * <p>Because the resolver is keyed by endpoint, a single downstream acknowledgement operator can
+ * serve receipt handles originating from multiple sources that point at different clusters with
+ * different credentials.
+ *
+ * <p>When a resolver class is configured, it takes precedence over the static {@code
+ * rocketmq.client.access-key} / {@code rocketmq.client.secret-key} options.
+ */
+@PublicEvolving
+public interface CredentialsResolver {
+
+ /**
+ * Called once right after instantiation with the operator configuration, before any {@link
+ * #resolve(String)} call. Implementations may read custom options from it.
+ *
+ * @param configuration the operator configuration.
+ */
+ default void configure(Configuration configuration) {}
+
+ /**
+ * Resolve the session credentials for the given proxy endpoint.
+ *
+ * @param endpoint the proxy endpoint the SDK client will connect to, e.g. {@code
+ * "127.0.0.1:8081"}.x
+ * @return the credentials provider for the endpoint, or {@code null} if the endpoint requires
+ * no authentication.
+ */
+ @Nullable
+ SessionCredentialsProvider resolve(String endpoint);
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolvers.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolvers.java
new file mode 100644
index 0000000..e46915c
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolvers.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.common;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.StringUtils;
+
+import org.apache.rocketmq.client.apis.ClientConfigurationBuilder;
+import org.apache.rocketmq.client.apis.SessionCredentialsProvider;
+import org.apache.rocketmq.client.apis.StaticSessionCredentialsProvider;
+
+import javax.annotation.Nullable;
+
+/** Instantiates {@link CredentialsResolver}s and applies credentials to SDK client builders. */
+@Internal
+public final class CredentialsResolvers {
+
+ private CredentialsResolvers() {}
+
+ /**
+ * Create the {@link CredentialsResolver} configured under {@link
+ * RocketMQGrpcOptions#CREDENTIALS_RESOLVER_CLASS}, or return {@code null} if none is
+ * configured. The resolver is instantiated reflectively and {@link
+ * CredentialsResolver#configure(Configuration) configured} before being returned.
+ */
+ @Nullable
+ public static CredentialsResolver createFromConfiguration(Configuration configuration) {
+ final String className = configuration.get(RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS);
+ if (StringUtils.isNullOrWhitespaceOnly(className)) {
+ return null;
+ }
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = CredentialsResolvers.class.getClassLoader();
+ }
+ try {
+ final CredentialsResolver resolver =
+ Class.forName(className, true, classLoader)
+ .asSubclass(CredentialsResolver.class)
+ .getDeclaredConstructor()
+ .newInstance();
+ resolver.configure(configuration);
+ return resolver;
+ } catch (ReflectiveOperationException | ClassCastException e) {
+ throw new FlinkRuntimeException(
+ "Failed to instantiate the credentials resolver '"
+ + className
+ + "'. It must be a public implementation of "
+ + CredentialsResolver.class.getName()
+ + " with a public no-argument constructor.",
+ e);
+ }
+ }
+
+ /**
+ * Apply credentials for the given endpoint to the SDK client builder. A configured {@link
+ * CredentialsResolver} takes precedence; otherwise the static access/secret key options are
+ * used when both are present.
+ */
+ public static void applyCredentials(
+ ClientConfigurationBuilder builder,
+ String endpoint,
+ Configuration configuration,
+ @Nullable CredentialsResolver resolver) {
+ if (resolver != null) {
+ final SessionCredentialsProvider provider = resolver.resolve(endpoint);
+ if (provider != null) {
+ builder.setCredentialProvider(provider);
+ }
+ return;
+ }
+ final String accessKey = configuration.get(RocketMQGrpcOptions.ACCESS_KEY);
+ final String secretKey = configuration.get(RocketMQGrpcOptions.SECRET_KEY);
+ if (!StringUtils.isNullOrWhitespaceOnly(accessKey)
+ && !StringUtils.isNullOrWhitespaceOnly(secretKey)) {
+ builder.setCredentialProvider(
+ new StaticSessionCredentialsProvider(accessKey, secretKey));
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/FilterExpressionParser.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/FilterExpressionParser.java
new file mode 100644
index 0000000..3ee0596
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/common/FilterExpressionParser.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.common;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.util.StringUtils;
+
+import org.apache.rocketmq.client.apis.consumer.FilterExpression;
+import org.apache.rocketmq.client.apis.consumer.FilterExpressionType;
+
+/**
+ * Parses a filter definition into an SDK {@link FilterExpression}. A {@code null} or blank
+ * expression matches all messages ({@link FilterExpression#SUB_ALL}).
+ */
+@Internal
+public class FilterExpressionParser {
+
+ private FilterExpressionParser() {}
+
+ /**
+ * Build a {@link FilterExpression} of the given type.
+ *
+ * @param expression the tag expression (e.g. {@code "tagA||tagB"}) or SQL92 expression; a blank
+ * value subscribes to all messages.
+ * @param type the filter type; a blank value defaults to {@link FilterExpressionType#TAG}.
+ */
+ public static FilterExpression parse(String expression, String type) {
+ if (StringUtils.isNullOrWhitespaceOnly(expression) || "*".equals(expression.trim())) {
+ return FilterExpression.SUB_ALL;
+ }
+ final FilterExpressionType filterType = parseType(type);
+ return new FilterExpression(expression.trim(), filterType);
+ }
+
+ private static FilterExpressionType parseType(String type) {
+ if (StringUtils.isNullOrWhitespaceOnly(type)) {
+ return FilterExpressionType.TAG;
+ }
+ switch (type.trim().toUpperCase()) {
+ case "SQL92":
+ return FilterExpressionType.SQL92;
+ case "TAG":
+ return FilterExpressionType.TAG;
+ default:
+ throw new IllegalArgumentException("Unsupported filter expression type: " + type);
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/ProducerProvider.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/ProducerProvider.java
new file mode 100644
index 0000000..5c59c58
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/ProducerProvider.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.sink;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.common.ClientConfigurationProvider;
+
+import org.apache.rocketmq.client.apis.ClientException;
+import org.apache.rocketmq.client.apis.ClientServiceProvider;
+import org.apache.rocketmq.client.apis.producer.Producer;
+import org.apache.rocketmq.client.apis.producer.ProducerBuilder;
+
+/** A factory that builds a gRPC {@link Producer} from a Flink configuration. */
+@Internal
+public class ProducerProvider {
+
+ private ProducerProvider() {}
+
+ /** Build a {@link Producer} using the given service provider and configuration. */
+ public static Producer create(ClientServiceProvider provider, Configuration configuration)
+ throws ClientException {
+ final ProducerBuilder builder =
+ provider.newProducerBuilder()
+ .setClientConfiguration(
+ ClientConfigurationProvider.getClientConfiguration(configuration))
+ .setMaxAttempts(configuration.get(RocketMQGrpcSinkOptions.MAX_ATTEMPTS));
+
+ final String topic = configuration.get(RocketMQGrpcSinkOptions.TOPIC);
+ if (topic != null && !topic.isEmpty()) {
+ builder.setTopics(topic);
+ }
+ return builder.build();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSink.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSink.java
new file mode 100644
index 0000000..67259ec
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSink.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.sink;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.api.connector.sink2.WriterInitContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.sink.serialization.RocketMQGrpcSerializationSchema;
+
+import java.io.IOException;
+
+/**
+ * The gRPC ({@code rocketmq-client-java}) implementation of a RocketMQ {@link Sink}. It provides
+ * at-least-once semantics using the synchronous {@code Producer.send()}; there is no committer and
+ * no two-phase transaction commit.
+ */
+@PublicEvolving
+public class RocketMQGrpcSink<IN> implements Sink<IN> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Configuration configuration;
+ private final RocketMQGrpcSerializationSchema<IN> serializationSchema;
+
+ RocketMQGrpcSink(
+ Configuration configuration, RocketMQGrpcSerializationSchema<IN> serializationSchema) {
+ this.configuration = configuration;
+ this.serializationSchema = serializationSchema;
+ }
+
+ /** Create a {@link RocketMQGrpcSinkBuilder} to construct a new {@link RocketMQGrpcSink}. */
+ public static <IN> RocketMQGrpcSinkBuilder<IN> builder() {
+ return new RocketMQGrpcSinkBuilder<>();
+ }
+
+ @Override
+ public SinkWriter<IN> createWriter(WriterInitContext context) throws IOException {
+ return new RocketMQGrpcSinkWriter<>(
+ configuration,
+ serializationSchema,
+ context.asSerializationSchemaInitializationContext());
+ }
+
+ @Deprecated
+ @Override
+ public SinkWriter<IN> createWriter(InitContext context) throws IOException {
+ return new RocketMQGrpcSinkWriter<>(
+ configuration,
+ serializationSchema,
+ context.asSerializationSchemaInitializationContext());
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilder.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilder.java
new file mode 100644
index 0000000..5b53c38
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilder.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.sink;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+import org.apache.flink.connector.rocketmq.grpc.sink.serialization.RocketMQGrpcSerializationSchema;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/** A fluent builder to construct a {@link RocketMQGrpcSink}. */
+@PublicEvolving
+public class RocketMQGrpcSinkBuilder<IN> {
+
+ private final Configuration configuration;
+ private RocketMQGrpcSerializationSchema<IN> serializationSchema;
+
+ public RocketMQGrpcSinkBuilder() {
+ this.configuration = new Configuration();
+ }
+
+ /** Set the gRPC access point (proxy) endpoints. */
+ public RocketMQGrpcSinkBuilder<IN> setEndpoints(String endpoints) {
+ return setConfig(RocketMQGrpcOptions.ENDPOINTS, endpoints);
+ }
+
+ /** Set the parent topic for value-only serialization. */
+ public RocketMQGrpcSinkBuilder<IN> setTopic(String topic) {
+ return setConfig(RocketMQGrpcSinkOptions.TOPIC, topic);
+ }
+
+ /** Set the lite (sub) topic attached to every record for value-only serialization. */
+ public RocketMQGrpcSinkBuilder<IN> setLiteTopic(String liteTopic) {
+ return setConfig(RocketMQGrpcSinkOptions.LITE_TOPIC, liteTopic);
+ }
+
+ /** Set the {@link RocketMQGrpcSerializationSchema}. */
+ public RocketMQGrpcSinkBuilder<IN> setSerializer(
+ RocketMQGrpcSerializationSchema<IN> serializationSchema) {
+ this.serializationSchema = checkNotNull(serializationSchema);
+ return this;
+ }
+
+ /**
+ * Set a value-only serializer that encodes the record into the message body and publishes it to
+ * the configured parent {@link RocketMQGrpcSinkOptions#TOPIC} carrying the configured {@link
+ * RocketMQGrpcSinkOptions#LITE_TOPIC}.
+ */
+ public RocketMQGrpcSinkBuilder<IN> setValueOnlySerializer(
+ SerializationSchema<IN> serializationSchema) {
+ final String topic = configuration.get(RocketMQGrpcSinkOptions.TOPIC);
+ checkNotNull(topic, "topic must be configured before setting a value-only serializer");
+ final String liteTopic = configuration.get(RocketMQGrpcSinkOptions.LITE_TOPIC);
+ checkNotNull(
+ liteTopic, "lite topic must be configured before setting a value-only serializer");
+ this.serializationSchema =
+ RocketMQGrpcSerializationSchema.flinkSchema(topic, liteTopic, serializationSchema);
+ return this;
+ }
+
+ /** Set an arbitrary configuration option. */
+ public <T> RocketMQGrpcSinkBuilder<IN> setConfig(ConfigOption<T> key, T value) {
+ configuration.set(key, value);
+ return this;
+ }
+
+ /** Add arbitrary configuration options. */
+ public RocketMQGrpcSinkBuilder<IN> setConfig(Configuration config) {
+ configuration.addAll(config);
+ return this;
+ }
+
+ /** Build the {@link RocketMQGrpcSink}. */
+ public RocketMQGrpcSink<IN> build() {
+ checkNotNull(
+ configuration.get(RocketMQGrpcOptions.ENDPOINTS), "endpoints must be configured");
+ checkNotNull(serializationSchema, "serializer must be configured");
+ return new RocketMQGrpcSink<>(configuration, serializationSchema);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkOptions.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkOptions.java
new file mode 100644
index 0000000..825d80b
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkOptions.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.sink;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+
+/**
+ * Configuration options for the RocketMQ gRPC {@code Producer} based sink. These options are
+ * combined with the shared {@link org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions}.
+ * The sink provides at-least-once semantics via synchronous {@code Producer.send()}.
+ *
+ * <p>These are the programmatic/SDK-facing options used by the sink builder and its runtime; every
+ * key carries the {@link #PRODUCER_CONFIG_PREFIX} prefix. The SQL DDL keys are defined separately
+ * in {@code RocketMQGrpcConnectorOptions}.
+ */
+@PublicEvolving
+public class RocketMQGrpcSinkOptions {
+
+ private RocketMQGrpcSinkOptions() {}
+
+ /** Prefix for the RocketMQ gRPC sink options. */
+ public static final String PRODUCER_CONFIG_PREFIX = "rocketmq.sink.";
+
+ public static final ConfigOption<String> TOPIC =
+ ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "topic")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The parent topic to send records to when using a value-only "
+ + "serializer.");
+
+ public static final ConfigOption<String> LITE_TOPIC =
+ ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "lite-topic")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The lite (sub) topic to attach to every record when using a "
+ + "value-only serializer. The message is published to the "
+ + "parent topic configured via '"
+ + PRODUCER_CONFIG_PREFIX
+ + "topic' and carries this lite topic, so that a "
+ + "LiteSimpleConsumer bound to the parent topic can receive "
+ + "it.");
+
+ public static final ConfigOption<Integer> MAX_ATTEMPTS =
+ ConfigOptions.key(PRODUCER_CONFIG_PREFIX + "max-attempts")
+ .intType()
+ .defaultValue(3)
+ .withDescription("The maximum number of send attempts for a message.");
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkWriter.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkWriter.java
new file mode 100644
index 0000000..041be68
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkWriter.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.sink;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.sink.serialization.RocketMQGrpcSerializationSchema;
+
+import org.apache.rocketmq.client.apis.ClientException;
+import org.apache.rocketmq.client.apis.ClientServiceProvider;
+import org.apache.rocketmq.client.apis.message.Message;
+import org.apache.rocketmq.client.apis.producer.Producer;
+
+import java.io.IOException;
+
+/**
+ * The {@link SinkWriter} that synchronously sends records to RocketMQ using the gRPC {@code
+ * Producer}. Each {@code send()} is a synchronous call, providing at-least-once semantics.
+ */
+@Internal
+public class RocketMQGrpcSinkWriter<IN> implements SinkWriter<IN> {
+
+ private final ClientServiceProvider provider;
+ private final Producer producer;
+ private final RocketMQGrpcSerializationSchema<IN> serializationSchema;
+
+ public RocketMQGrpcSinkWriter(
+ Configuration configuration,
+ RocketMQGrpcSerializationSchema<IN> serializationSchema,
+ SerializationSchema.InitializationContext initializationContext)
+ throws IOException {
+ this.provider = ClientServiceProvider.loadService();
+ this.serializationSchema = serializationSchema;
+ try {
+ this.serializationSchema.open(initializationContext);
+ this.producer = ProducerProvider.create(provider, configuration);
+ } catch (Exception e) {
+ throw new IOException("Failed to initialize the RocketMQ gRPC producer.", e);
+ }
+ }
+
+ @Override
+ public void write(IN element, Context context) throws IOException {
+ final Long timestamp = context.timestamp();
+ final Message message =
+ serializationSchema.serialize(element, provider.newMessageBuilder(), timestamp);
+ try {
+ producer.send(message);
+ } catch (ClientException e) {
+ throw new IOException("Failed to send message to RocketMQ.", e);
+ }
+ }
+
+ @Override
+ public void flush(boolean endOfInput) {
+ // Messages are sent synchronously, so there is nothing to flush.
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (producer != null) {
+ producer.close();
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchema.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchema.java
new file mode 100644
index 0000000..1e13f25
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchema.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.sink.serialization;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+
+import org.apache.rocketmq.client.apis.message.Message;
+import org.apache.rocketmq.client.apis.message.MessageBuilder;
+
+import java.io.Serializable;
+
+/**
+ * A serialization schema that converts a value of type {@code T} into a RocketMQ gRPC {@link
+ * Message} using the given {@link MessageBuilder}.
+ *
+ * @param <T> the type of values being serialized
+ */
+@PublicEvolving
+public interface RocketMQGrpcSerializationSchema<T> extends Serializable {
+
+ /**
+ * Initialization method for the schema. It is called before the actual working method {@link
+ * #serialize} and thus suitable for one time setup work.
+ *
+ * @param context Contextual information that can be used during initialization.
+ */
+ default void open(SerializationSchema.InitializationContext context) throws Exception {
+ // Nothing to do here for the default implementation.
+ }
+
+ /**
+ * Serialize the given element into a {@link Message}.
+ *
+ * @param element the element to serialize.
+ * @param messageBuilder a fresh message builder to construct the message with.
+ * @param timestamp the (nullable) event timestamp of the element.
+ * @return the RocketMQ {@link Message} to send.
+ */
+ Message serialize(T element, MessageBuilder messageBuilder, Long timestamp);
+
+ /**
+ * Create a {@link RocketMQGrpcSerializationSchema} by wrapping a Flink {@link
+ * SerializationSchema}. The value is serialized into the message body and published to the
+ * given parent {@code topic} carrying the given {@code liteTopic}, so that a {@code
+ * LiteSimpleConsumer} bound to the parent topic can receive it.
+ */
+ static <T> RocketMQGrpcSerializationSchema<T> flinkSchema(
+ String topic, String liteTopic, SerializationSchema<T> serializationSchema) {
+ return new RocketMQGrpcSerializationSchemaWrapper<>(topic, liteTopic, serializationSchema);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchemaWrapper.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchemaWrapper.java
new file mode 100644
index 0000000..061408b
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/sink/serialization/RocketMQGrpcSerializationSchemaWrapper.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.sink.serialization;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+
+import org.apache.rocketmq.client.apis.message.Message;
+import org.apache.rocketmq.client.apis.message.MessageBuilder;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A {@link RocketMQGrpcSerializationSchema} that adapts a Flink {@link SerializationSchema} by
+ * serializing the value into the message body and publishing it to a fixed parent topic carrying a
+ * fixed lite (sub) topic.
+ */
+@Internal
+public class RocketMQGrpcSerializationSchemaWrapper<T>
+ implements RocketMQGrpcSerializationSchema<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String topic;
+ private final String liteTopic;
+ private final SerializationSchema<T> serializationSchema;
+
+ public RocketMQGrpcSerializationSchemaWrapper(
+ String topic, String liteTopic, SerializationSchema<T> serializationSchema) {
+ this.topic = checkNotNull(topic, "topic must not be null");
+ this.liteTopic = checkNotNull(liteTopic, "lite topic must not be null");
+ this.serializationSchema = checkNotNull(serializationSchema);
+ }
+
+ @Override
+ public void open(SerializationSchema.InitializationContext context) throws Exception {
+ serializationSchema.open(context);
+ }
+
+ @Override
+ public Message serialize(T element, MessageBuilder messageBuilder, Long timestamp) {
+ return messageBuilder
+ .setTopic(topic)
+ .setLiteTopic(liteTopic)
+ .setBody(serializationSchema.serialize(element))
+ .build();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicies.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicies.java
new file mode 100644
index 0000000..a85891f
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicies.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.FlinkRuntimeException;
+import org.apache.flink.util.StringUtils;
+
+import javax.annotation.Nullable;
+
+/** Instantiates {@link InvisibleDurationRenewalPolicy} implementations reflectively. */
+@Internal
+public final class InvisibleDurationRenewalPolicies {
+
+ private InvisibleDurationRenewalPolicies() {}
+
+ /**
+ * Create the {@link InvisibleDurationRenewalPolicy} configured under {@link
+ * RocketMQGrpcSourceOptions#RENEWAL_POLICY_CLASS}, or return {@code null} if none is
+ * configured. The policy is instantiated reflectively and {@link
+ * InvisibleDurationRenewalPolicy#configure(Configuration) configured} before being returned.
+ */
+ @Nullable
+ public static InvisibleDurationRenewalPolicy createFromConfiguration(
+ Configuration configuration) {
+ final String className = configuration.get(RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS);
+ if (StringUtils.isNullOrWhitespaceOnly(className)) {
+ return null;
+ }
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = InvisibleDurationRenewalPolicies.class.getClassLoader();
+ }
+ try {
+ final InvisibleDurationRenewalPolicy policy =
+ Class.forName(className, true, classLoader)
+ .asSubclass(InvisibleDurationRenewalPolicy.class)
+ .getDeclaredConstructor()
+ .newInstance();
+ policy.configure(configuration);
+ return policy;
+ } catch (ReflectiveOperationException | ClassCastException e) {
+ throw new FlinkRuntimeException(
+ "Failed to instantiate the invisible duration renewal policy '"
+ + className
+ + "'. It must be a public implementation of "
+ + InvisibleDurationRenewalPolicy.class.getName()
+ + " with a public no-argument constructor.",
+ e);
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicy.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicy.java
new file mode 100644
index 0000000..60ead32
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPolicy.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.time.Duration;
+
+/**
+ * A user-provided policy that decides whether a received message that is still buffered inside the
+ * source (i.e. not yet emitted downstream) should have its invisible duration renewed.
+ *
+ * <p>The source invokes {@link #renew(MessageView, int)} {@link
+ * RocketMQGrpcSourceOptions#RENEWAL_AHEAD_TIME} ahead of the moment the message would become
+ * visible again (for example with a 60s invisible duration and a 5s ahead time, the policy is
+ * consulted 55s after the message was received). Returning a positive duration triggers a {@code
+ * changeInvisibleDuration} call with that duration and schedules the next consultation; returning
+ * {@code null} (or a non-positive duration) stops renewing so the message becomes visible again
+ * after the current invisible duration elapses.
+ *
+ * <p>Messages that have already been handed to the record emitter are never renewed, because
+ * renewing refreshes the receipt handle and would invalidate the handle travelling downstream.
+ *
+ * <p>Implementations are instantiated reflectively from {@link
+ * RocketMQGrpcSourceOptions#RENEWAL_POLICY_CLASS} and therefore need a public no-argument
+ * constructor. One instance is created per split reader; invocations happen on a single renewal
+ * thread.
+ */
+@PublicEvolving
+public interface InvisibleDurationRenewalPolicy extends Serializable {
+
+ /**
+ * Configure this policy with the source configuration. Called once directly after
+ * instantiation.
+ */
+ default void configure(Configuration configuration) {}
+
+ /**
+ * Decide whether to renew the invisible duration of the given message.
+ *
+ * @param messageView the message still buffered inside the source.
+ * @param renewalCount how many times this message has already been renewed; {@code 0} on the
+ * first consultation.
+ * @return the new invisible duration, or {@code null} to stop renewing.
+ */
+ @Nullable
+ Duration renew(MessageView messageView, int renewalCount);
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSource.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSource.java
new file mode 100644
index 0000000..1f21b86
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSource.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.api.connector.source.Source;
+import org.apache.flink.api.connector.source.SourceReader;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.api.connector.source.SplitEnumerator;
+import org.apache.flink.api.connector.source.SplitEnumeratorContext;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
+import org.apache.flink.connector.base.source.reader.splitreader.SplitReader;
+import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessage;
+import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessageTypeInfo;
+import org.apache.flink.connector.rocketmq.grpc.source.deserialization.RocketMQGrpcDeserializationSchema;
+import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumState;
+import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumStateSerializer;
+import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumerator;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageViewImpl;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.RocketMQGrpcSourceFetcherManager;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.RocketMQGrpcSourceReader;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.RocketMQGrpcSourceRecordEmitter;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.RocketMQGrpcSourceSplitReader;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplitSerializer;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.util.UserCodeClassLoader;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.function.Supplier;
+
+/** The gRPC ({@code rocketmq-client-java}) implementation of a FLIP-27 RocketMQ {@link Source}. */
+@PublicEvolving
+public class RocketMQGrpcSource<OUT>
+ implements Source<
+ AckableMessage<OUT>, RocketMQGrpcSourceSplit, RocketMQGrpcSourceEnumState>,
+ ResultTypeQueryable<AckableMessage<OUT>> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG = LoggerFactory.getLogger(RocketMQGrpcSource.class);
+
+ private final Configuration configuration;
+ private final Boundedness boundedness;
+ private final RocketMQGrpcDeserializationSchema<OUT> deserializationSchema;
+
+ RocketMQGrpcSource(
+ Configuration configuration,
+ Boundedness boundedness,
+ RocketMQGrpcDeserializationSchema<OUT> deserializationSchema) {
+ this.configuration = configuration;
+ this.boundedness = boundedness;
+ this.deserializationSchema = deserializationSchema;
+ }
+
+ /** Get a {@link RocketMQGrpcSourceBuilder} to build a {@link RocketMQGrpcSource}. */
+ public static <OUT> RocketMQGrpcSourceBuilder<OUT> builder() {
+ return new RocketMQGrpcSourceBuilder<>();
+ }
+
+ @Override
+ public Boundedness getBoundedness() {
+ return boundedness;
+ }
+
+ @Override
+ public SourceReader<AckableMessage<OUT>, RocketMQGrpcSourceSplit> createReader(
+ SourceReaderContext readerContext) throws Exception {
+ final FutureCompletingBlockingQueue<RecordsWithSplitIds<MessageViewImpl>> elementsQueue =
+ new FutureCompletingBlockingQueue<>();
+
+ deserializationSchema.open(
+ new DeserializationSchema.InitializationContext() {
+ @Override
+ public MetricGroup getMetricGroup() {
+ return readerContext.metricGroup().addGroup("deserializer");
+ }
+
+ @Override
+ public UserCodeClassLoader getUserCodeClassLoader() {
+ return readerContext.getUserCodeClassLoader();
+ }
+ });
+
+ final RocketMQGrpcSourceSplitReader splitReader =
+ new RocketMQGrpcSourceSplitReader(configuration);
+ final Supplier<SplitReader<MessageViewImpl, RocketMQGrpcSourceSplit>> splitReaderSupplier =
+ () -> splitReader;
+
+ final RocketMQGrpcSourceFetcherManager fetcherManager =
+ new RocketMQGrpcSourceFetcherManager(elementsQueue, splitReaderSupplier);
+
+ final RocketMQGrpcSourceRecordEmitter<OUT> recordEmitter =
+ new RocketMQGrpcSourceRecordEmitter<>(
+ deserializationSchema,
+ configuration.get(RocketMQGrpcOptions.NAMESPACE),
+ configuration.get(RocketMQGrpcSourceOptions.CONSUMER_GROUP));
+
+ final RocketMQGrpcSourceReader<OUT> reader =
+ new RocketMQGrpcSourceReader<>(
+ elementsQueue, fetcherManager, recordEmitter, configuration, readerContext);
+
+ return reader;
+ }
+
+ @Override
+ public SplitEnumerator<RocketMQGrpcSourceSplit, RocketMQGrpcSourceEnumState> createEnumerator(
+ SplitEnumeratorContext<RocketMQGrpcSourceSplit> enumContext) {
+ return new RocketMQGrpcSourceEnumerator(enumContext);
+ }
+
+ @Override
+ public SplitEnumerator<RocketMQGrpcSourceSplit, RocketMQGrpcSourceEnumState> restoreEnumerator(
+ SplitEnumeratorContext<RocketMQGrpcSourceSplit> enumContext,
+ RocketMQGrpcSourceEnumState checkpoint) {
+ LOG.info(
+ "Restoring RocketMQ gRPC source enumerator from checkpoint; the Pop-based "
+ + "enumerator is stateless, so no split assignment state is recovered.");
+ return new RocketMQGrpcSourceEnumerator(enumContext);
+ }
+
+ @Override
+ public SimpleVersionedSerializer<RocketMQGrpcSourceSplit> getSplitSerializer() {
+ return new RocketMQGrpcSourceSplitSerializer();
+ }
+
+ @Override
+ public SimpleVersionedSerializer<RocketMQGrpcSourceEnumState>
+ getEnumeratorCheckpointSerializer() {
+ return new RocketMQGrpcSourceEnumStateSerializer();
+ }
+
+ @Override
+ public TypeInformation<AckableMessage<OUT>> getProducedType() {
+ return new AckableMessageTypeInfo<>(deserializationSchema.getProducedType());
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilder.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilder.java
new file mode 100644
index 0000000..f38dcf8
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilder.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+import org.apache.flink.connector.rocketmq.grpc.source.deserialization.RocketMQGrpcDeserializationSchema;
+
+import java.time.Duration;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/** A fluent builder to construct a {@link RocketMQGrpcSource}. */
+@PublicEvolving
+public class RocketMQGrpcSourceBuilder<OUT> {
+
+ private final Configuration configuration;
+ private String mainTopic;
+ private Boundedness boundedness;
+ private RocketMQGrpcDeserializationSchema<OUT> deserializationSchema;
+
+ public RocketMQGrpcSourceBuilder() {
+ this.configuration = new Configuration();
+ this.boundedness = Boundedness.CONTINUOUS_UNBOUNDED;
+ }
+
+ /** Set the gRPC access point (proxy) endpoints. */
+ public RocketMQGrpcSourceBuilder<OUT> setEndpoints(String endpoints) {
+ return setConfig(RocketMQGrpcOptions.ENDPOINTS, endpoints);
+ }
+
+ /** Set the consumer group of the LiteSimpleConsumer. */
+ public RocketMQGrpcSourceBuilder<OUT> setConsumerGroup(String consumerGroup) {
+ return setConfig(RocketMQGrpcSourceOptions.CONSUMER_GROUP, consumerGroup);
+ }
+
+ /** Set the main lite topic bound by the LiteSimpleConsumer. Every subtask binds this topic. */
+ public RocketMQGrpcSourceBuilder<OUT> setMainTopic(String mainTopic) {
+ checkArgument(
+ mainTopic != null && !mainTopic.trim().isEmpty(),
+ "main topic must not be null or blank");
+ this.mainTopic = mainTopic;
+ return this;
+ }
+
+ /**
+ * Set the fetch concurrency of each subtask, i.e. the number of concurrent fetch requests it
+ * issues. The requests are served by worker threads sharing a single thread-safe {@code
+ * SimpleConsumer}. Defaults to {@code 1}.
+ */
+ public RocketMQGrpcSourceBuilder<OUT> setFetchConcurrency(int fetchConcurrency) {
+ return setConfig(RocketMQGrpcSourceOptions.FETCH_CONCURRENCY, fetchConcurrency);
+ }
+
+ /**
+ * Set the fully qualified class name of an {@link InvisibleDurationRenewalPolicy}. When
+ * configured, the source consults the policy shortly before a still-buffered message would
+ * become visible again, and the policy decides whether to extend its invisible duration.
+ */
+ public RocketMQGrpcSourceBuilder<OUT> setRenewalPolicyClass(String renewalPolicyClass) {
+ return setConfig(RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS, renewalPolicyClass);
+ }
+
+ /**
+ * Set how long before a buffered message becomes visible again the renewal policy is consulted.
+ * Defaults to 5 seconds, i.e. with a 60s invisible duration the policy runs 55s after the
+ * message was received.
+ */
+ public RocketMQGrpcSourceBuilder<OUT> setRenewalAheadTime(Duration aheadTime) {
+ return setConfig(RocketMQGrpcSourceOptions.RENEWAL_AHEAD_TIME, aheadTime);
+ }
+
+ /** Set the boundedness of this source. Defaults to {@link Boundedness#CONTINUOUS_UNBOUNDED}. */
+ public RocketMQGrpcSourceBuilder<OUT> setBoundedness(Boundedness boundedness) {
+ this.boundedness = checkNotNull(boundedness);
+ return this;
+ }
+
+ /** Set the {@link RocketMQGrpcDeserializationSchema}. */
+ public RocketMQGrpcSourceBuilder<OUT> setDeserializer(
+ RocketMQGrpcDeserializationSchema<OUT> deserializationSchema) {
+ this.deserializationSchema = checkNotNull(deserializationSchema);
+ return this;
+ }
+
+ /** Set a value-only deserializer that decodes the message body using a Flink schema. */
+ public RocketMQGrpcSourceBuilder<OUT> setValueOnlyDeserializer(
+ DeserializationSchema<OUT> deserializationSchema) {
+ this.deserializationSchema =
+ RocketMQGrpcDeserializationSchema.flinkSchema(deserializationSchema);
+ return this;
+ }
+
+ /** Set an arbitrary configuration option. */
+ public <T> RocketMQGrpcSourceBuilder<OUT> setConfig(ConfigOption<T> key, T value) {
+ configuration.set(key, value);
+ return this;
+ }
+
+ /** Add arbitrary configuration options. */
+ public RocketMQGrpcSourceBuilder<OUT> setConfig(Configuration config) {
+ configuration.addAll(config);
+ return this;
+ }
+
+ /** Build the {@link RocketMQGrpcSource}. */
+ public RocketMQGrpcSource<OUT> build() {
+ checkNotNull(
+ configuration.get(RocketMQGrpcOptions.ENDPOINTS), "endpoints must be configured");
+ checkNotNull(
+ configuration.get(RocketMQGrpcSourceOptions.CONSUMER_GROUP),
+ "consumer group must be configured");
+ checkArgument(
+ mainTopic != null && !mainTopic.trim().isEmpty(),
+ "the main topic must be configured");
+ checkArgument(
+ configuration.get(RocketMQGrpcSourceOptions.FETCH_CONCURRENCY) >= 1,
+ "fetch concurrency must be at least 1");
+ checkNotNull(deserializationSchema, "deserializer must be configured");
+ configuration.set(RocketMQGrpcSourceOptions.MAIN_TOPIC, mainTopic);
+ return new RocketMQGrpcSource<>(configuration, boundedness, deserializationSchema);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceOptions.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceOptions.java
new file mode 100644
index 0000000..94b1e5a
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceOptions.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+
+import java.time.Duration;
+
+/**
+ * Configuration options for the RocketMQ gRPC {@code SimpleConsumer} based source. These options
+ * are combined with the shared {@link
+ * org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions}.
+ *
+ * <p>These are the programmatic/SDK-facing options used by the source builder and its runtime;
+ * every key carries the {@link #CONSUMER_CONFIG_PREFIX} prefix. The SQL DDL keys are defined
+ * separately in {@code RocketMQGrpcConnectorOptions}.
+ */
+@PublicEvolving
+public class RocketMQGrpcSourceOptions {
+
+ private RocketMQGrpcSourceOptions() {}
+
+ /** Prefix for the RocketMQ gRPC source options. */
+ public static final String CONSUMER_CONFIG_PREFIX = "rocketmq.source.";
+
+ public static final ConfigOption<String> CONSUMER_GROUP =
+ ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "consumer-group")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The consumer group of the SimpleConsumer.");
+
+ public static final ConfigOption<String> MAIN_TOPIC =
+ ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "main-topic")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The main lite topic bound by the LiteSimpleConsumer. Every subtask "
+ + "binds this topic; the broker performs message-level load "
+ + "balancing across the consumer group.");
+
+ public static final ConfigOption<Integer> FETCH_CONCURRENCY =
+ ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "fetch-concurrency")
+ .intType()
+ .defaultValue(1)
+ .withDescription(
+ "The fetch concurrency of each subtask, i.e. the number of concurrent "
+ + "fetch requests it issues. The requests are served by worker "
+ + "threads sharing a single thread-safe SimpleConsumer, so "
+ + "increasing this raises a single subtask's fetch throughput "
+ + "without changing the Flink parallelism.");
+
+ public static final ConfigOption<Duration> AWAIT_DURATION =
+ ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "await-duration")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(20))
+ .withDescription(
+ "The long-polling await duration of a single receive() invocation.");
+
+ public static final ConfigOption<Duration> INVISIBLE_DURATION =
+ ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "invisible-duration")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(60))
+ .withDescription(
+ "The invisible duration of a received message. Because messages are "
+ + "acknowledged by a downstream operator, it must be larger "
+ + "than the full downstream processing time of a message so "
+ + "that a message is not prematurely redelivered while still "
+ + "in flight; un-acked messages are redelivered after this "
+ + "duration, which provides the at-least-once guarantee.");
+
+ public static final ConfigOption<Integer> MAX_MESSAGE_NUM =
+ ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "max-message-num")
+ .intType()
+ .defaultValue(32)
+ .withDescription("The maximum number of messages returned by receive().");
+
+ public static final ConfigOption<String> RENEWAL_POLICY_CLASS =
+ ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "renewal-policy-class")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The fully qualified class name of an InvisibleDurationRenewalPolicy "
+ + "implementation. When configured, the source consults the "
+ + "policy 'renewal-ahead-time' before a still-buffered message "
+ + "would become visible again, and the policy decides whether "
+ + "to extend its invisible duration. When absent, no renewal "
+ + "is performed.");
+
+ public static final ConfigOption<Duration> RENEWAL_AHEAD_TIME =
+ ConfigOptions.key(CONSUMER_CONFIG_PREFIX + "renewal-ahead-time")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(5))
+ .withDescription(
+ "How long before a buffered message becomes visible again the renewal "
+ + "policy is consulted. For example with a 60s invisible "
+ + "duration and a 5s ahead time the policy runs 55s after the "
+ + "message was received. Must be positive and smaller than "
+ + "the invisible duration. Only effective when "
+ + "'renewal-policy-class' is configured.");
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchema.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchema.java
new file mode 100644
index 0000000..d9dbf07
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchema.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.deserialization;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView;
+import org.apache.flink.util.Collector;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+/** An interface for the deserialization of RocketMQ gRPC messages. */
+@PublicEvolving
+public interface RocketMQGrpcDeserializationSchema<T> extends Serializable, ResultTypeQueryable<T> {
+
+ /**
+ * Initialization method for the schema. It is called before the actual working method {@link
+ * #deserialize} and thus suitable for one time setup work.
+ *
+ * @param context Contextual information that can be used during initialization.
+ */
+ default void open(DeserializationSchema.InitializationContext context) throws Exception {
+ // Nothing to do here for the default implementation.
+ }
+
+ /**
+ * Deserializes a {@link MessageView} and outputs zero or more records through the {@link
+ * Collector}.
+ *
+ * @param messageView The MessageView to deserialize.
+ * @param out The collector to put the resulting records.
+ */
+ void deserialize(MessageView messageView, Collector<T> out) throws IOException;
+
+ /**
+ * Create a {@link RocketMQGrpcDeserializationSchema} by wrapping a Flink {@link
+ * DeserializationSchema}. The message body is deserialized using the given schema; the other
+ * fields such as key, tag and properties are ignored.
+ */
+ static <T> RocketMQGrpcDeserializationSchema<T> flinkSchema(
+ DeserializationSchema<T> deserializationSchema) {
+ return new RocketMQGrpcDeserializationSchemaWrapper<>(deserializationSchema);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchemaWrapper.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchemaWrapper.java
new file mode 100644
index 0000000..191cd29
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/deserialization/RocketMQGrpcDeserializationSchemaWrapper.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.deserialization;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView;
+import org.apache.flink.util.Collector;
+
+import java.io.IOException;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * A {@link RocketMQGrpcDeserializationSchema} that adapts a Flink {@link DeserializationSchema} by
+ * deserializing the message body only.
+ */
+@Internal
+public class RocketMQGrpcDeserializationSchemaWrapper<T>
+ implements RocketMQGrpcDeserializationSchema<T> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final DeserializationSchema<T> deserializationSchema;
+
+ public RocketMQGrpcDeserializationSchemaWrapper(
+ DeserializationSchema<T> deserializationSchema) {
+ this.deserializationSchema = checkNotNull(deserializationSchema);
+ }
+
+ @Override
+ public void open(DeserializationSchema.InitializationContext context) throws Exception {
+ deserializationSchema.open(context);
+ }
+
+ @Override
+ public void deserialize(MessageView messageView, Collector<T> out) throws IOException {
+ deserializationSchema.deserialize(messageView.getBody(), out);
+ }
+
+ @Override
+ public TypeInformation<T> getProducedType() {
+ return deserializationSchema.getProducedType();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumState.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumState.java
new file mode 100644
index 0000000..2f9422d
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumState.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.enumerator;
+
+/**
+ * The checkpointed state of the {@link RocketMQGrpcSourceEnumerator}.
+ *
+ * <p>The Pop model needs no enumerator state: splits carry no offset and every reader is simply
+ * assigned a placeholder split so it starts consuming all topics. This type only exists because the
+ * FLIP-27 {@code Source} API requires an enumerator checkpoint type, so it is a stateless marker.
+ */
+public class RocketMQGrpcSourceEnumState {}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumStateSerializer.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumStateSerializer.java
new file mode 100644
index 0000000..7e4faf6
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumStateSerializer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.enumerator;
+
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+
+/**
+ * The {@link SimpleVersionedSerializer} for {@link RocketMQGrpcSourceEnumState}. The state is a
+ * stateless marker, so serialization is empty.
+ */
+public class RocketMQGrpcSourceEnumStateSerializer
+ implements SimpleVersionedSerializer<RocketMQGrpcSourceEnumState> {
+
+ private static final int CURRENT_VERSION = 2;
+
+ @Override
+ public int getVersion() {
+ return CURRENT_VERSION;
+ }
+
+ @Override
+ public byte[] serialize(RocketMQGrpcSourceEnumState state) {
+ return new byte[0];
+ }
+
+ @Override
+ public RocketMQGrpcSourceEnumState deserialize(int version, byte[] serialized) {
+ return new RocketMQGrpcSourceEnumState();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumerator.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumerator.java
new file mode 100644
index 0000000..20c12a3
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/enumerator/RocketMQGrpcSourceEnumerator.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.enumerator;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.connector.source.SplitEnumerator;
+import org.apache.flink.api.connector.source.SplitEnumeratorContext;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+/**
+ * The split enumerator for the RocketMQ gRPC source.
+ *
+ * <p>The Pop model performs message-level load balancing on the broker side, so there is nothing to
+ * distribute: every reader consumes all configured topics and the broker spreads messages across
+ * the consumer group. The enumerator is therefore stateless and simply hands each registered reader
+ * a single placeholder split, which is what triggers that reader to start its pop loop.
+ */
+@Internal
+public class RocketMQGrpcSourceEnumerator
+ implements SplitEnumerator<RocketMQGrpcSourceSplit, RocketMQGrpcSourceEnumState> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RocketMQGrpcSourceEnumerator.class);
+
+ private final SplitEnumeratorContext<RocketMQGrpcSourceSplit> context;
+
+ public RocketMQGrpcSourceEnumerator(SplitEnumeratorContext<RocketMQGrpcSourceSplit> context) {
+ this.context = context;
+ }
+
+ @Override
+ public void start() {
+ // No topic discovery or split distribution is required for the Pop model.
+ }
+
+ @Override
+ public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) {
+ // Splits are assigned proactively when a reader registers.
+ }
+
+ @Override
+ public void addReader(int subtaskId) {
+ context.assignSplit(new RocketMQGrpcSourceSplit(), subtaskId);
+ context.signalNoMoreSplits(subtaskId);
+ LOG.info("Assigned the placeholder split to reader {}", subtaskId);
+ }
+
+ @Override
+ public void addSplitsBack(List<RocketMQGrpcSourceSplit> splits, int subtaskId) {
+ // The placeholder split is stateless and is re-assigned when the reader re-registers, so
+ // returned splits can be dropped.
+ LOG.info("RocketMQ gRPC source dropped returned placeholder splits: {}", splits);
+ }
+
+ @Override
+ public RocketMQGrpcSourceEnumState snapshotState(long checkpointId) {
+ return new RocketMQGrpcSourceEnumState();
+ }
+
+ @Override
+ public void close() {
+ // Nothing to close.
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/LiteSimpleConsumerProvider.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/LiteSimpleConsumerProvider.java
new file mode 100644
index 0000000..82b0c8d
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/LiteSimpleConsumerProvider.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.reader;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.common.ClientConfigurationProvider;
+import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions;
+
+import org.apache.rocketmq.client.apis.ClientException;
+import org.apache.rocketmq.client.apis.ClientServiceProvider;
+import org.apache.rocketmq.client.apis.consumer.LiteSimpleConsumer;
+
+/**
+ * A factory that builds a gRPC {@link LiteSimpleConsumer} (Pop model) from a Flink configuration.
+ *
+ * <p>The consumer binds the configured main topic and relies on its wildcard (generalized)
+ * subscription to receive messages from all of its sub topics; the consumer group must carry the
+ * {@code lite.sub.wildcard=true} attribute on the broker.
+ */
+@Internal
+public class LiteSimpleConsumerProvider {
+
+ private LiteSimpleConsumerProvider() {}
+
+ /**
+ * Build and start a {@link LiteSimpleConsumer} from the given Flink configuration.
+ *
+ * @param configuration the Flink configuration.
+ * @return a started {@link LiteSimpleConsumer}.
+ */
+ public static LiteSimpleConsumer create(Configuration configuration) throws ClientException {
+ final ClientServiceProvider provider = ClientServiceProvider.loadService();
+ return provider.newLiteSimpleConsumerBuilder()
+ .setClientConfiguration(
+ ClientConfigurationProvider.getClientConfiguration(configuration))
+ .setConsumerGroup(configuration.get(RocketMQGrpcSourceOptions.CONSUMER_GROUP))
+ .setAwaitDuration(configuration.get(RocketMQGrpcSourceOptions.AWAIT_DURATION))
+ .bindTopic(configuration.get(RocketMQGrpcSourceOptions.MAIN_TOPIC))
+ .build();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageView.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageView.java
new file mode 100644
index 0000000..747a1c7
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageView.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.reader;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * A read-only view over a RocketMQ message returned by the gRPC {@code SimpleConsumer}. It exposes
+ * the message attributes needed for deserialization while hiding the SDK receipt handle used for
+ * acknowledgement.
+ */
+public interface MessageView {
+
+ /** Get the unique message ID. */
+ String getMessageId();
+
+ /** Get the topic that the message belongs to. */
+ String getTopic();
+
+ /** Get the tag of the message, or {@code null} if the message has no tag. */
+ String getTag();
+
+ /** Get the keys of the message. */
+ Collection<String> getKeys();
+
+ /** Get the body of the message. */
+ byte[] getBody();
+
+ /** Get the number of times the message has been delivered. */
+ int getDeliveryAttempt();
+
+ /** Get the born timestamp of the message, used as the event time. */
+ long getEventTime();
+
+ /** Get the user-defined properties of the message. */
+ Map<String, String> getProperties();
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImpl.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImpl.java
new file mode 100644
index 0000000..d335139
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImpl.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.reader;
+
+import org.apache.flink.annotation.Internal;
+
+import org.apache.rocketmq.client.apis.message.MessageId;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.Future;
+
+/**
+ * The {@link MessageView} implementation backed by an SDK {@link
+ * org.apache.rocketmq.client.apis.message.MessageView}. It retains the underlying SDK message so
+ * that the reader can acknowledge it after a checkpoint completes.
+ *
+ * <p>It additionally tracks whether the message has been handed to the record emitter. Renewing the
+ * invisible duration refreshes the SDK receipt handle, so a renewal and the handle extraction at
+ * emit time must be mutually exclusive: both happen under the monitor of this instance, and once
+ * {@link #markEmitted()} has been called no further renewal is attempted.
+ */
+@Internal
+public class MessageViewImpl implements MessageView {
+
+ private final org.apache.rocketmq.client.apis.message.MessageView messageView;
+ private final byte[] body;
+
+ private boolean emitted;
+ private Future<?> renewalFuture;
+
+ public MessageViewImpl(org.apache.rocketmq.client.apis.message.MessageView messageView) {
+ this.messageView = messageView;
+ final ByteBuffer buffer = messageView.getBody();
+ this.body = new byte[buffer.remaining()];
+ buffer.get(this.body);
+ }
+
+ /** The underlying SDK message, used for acknowledgement via the {@code SimpleConsumer}. */
+ public org.apache.rocketmq.client.apis.message.MessageView getMessageView() {
+ return messageView;
+ }
+
+ /**
+ * Mark this message as handed to the record emitter and cancel any pending renewal. Blocks
+ * while a renewal RPC for this message is in flight so that the receipt handle extracted
+ * afterwards is stable.
+ */
+ public synchronized void markEmitted() {
+ emitted = true;
+ if (renewalFuture != null) {
+ renewalFuture.cancel(false);
+ renewalFuture = null;
+ }
+ }
+
+ /** Whether this message has been handed to the record emitter. */
+ public synchronized boolean isEmitted() {
+ return emitted;
+ }
+
+ /** Track the pending renewal task so that {@link #markEmitted()} can cancel it. */
+ public synchronized void setRenewalFuture(Future<?> future) {
+ if (emitted) {
+ future.cancel(false);
+ } else {
+ this.renewalFuture = future;
+ }
+ }
+
+ @Override
+ public String getMessageId() {
+ final MessageId messageId = messageView.getMessageId();
+ return messageId == null ? null : messageId.toString();
+ }
+
+ @Override
+ public String getTopic() {
+ return messageView.getTopic();
+ }
+
+ @Override
+ public String getTag() {
+ return messageView.getTag().orElse(null);
+ }
+
+ @Override
+ public Collection<String> getKeys() {
+ return messageView.getKeys();
+ }
+
+ @Override
+ public byte[] getBody() {
+ return body;
+ }
+
+ @Override
+ public int getDeliveryAttempt() {
+ return messageView.getDeliveryAttempt();
+ }
+
+ @Override
+ public long getEventTime() {
+ return messageView.getBornTimestamp();
+ }
+
+ @Override
+ public Map<String, String> getProperties() {
+ return messageView.getProperties();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceFetcherManager.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceFetcherManager.java
new file mode 100644
index 0000000..10ebd61
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceFetcherManager.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.reader;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
+import org.apache.flink.connector.base.source.reader.fetcher.SingleThreadFetcherManager;
+import org.apache.flink.connector.base.source.reader.splitreader.SplitReader;
+import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit;
+
+import java.util.function.Supplier;
+
+/**
+ * The {@link SingleThreadFetcherManager} for the RocketMQ gRPC source. The Pop source does not
+ * acknowledge messages from the source side (acknowledgement is deferred to a downstream operator),
+ * so this manager only provides a single fetcher thread that receives messages.
+ */
+@Internal
+public class RocketMQGrpcSourceFetcherManager
+ extends SingleThreadFetcherManager<MessageViewImpl, RocketMQGrpcSourceSplit> {
+
+ public RocketMQGrpcSourceFetcherManager(
+ FutureCompletingBlockingQueue<RecordsWithSplitIds<MessageViewImpl>> elementsQueue,
+ Supplier<SplitReader<MessageViewImpl, RocketMQGrpcSourceSplit>> splitReaderSupplier) {
+ super(elementsQueue, splitReaderSupplier, new Configuration());
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceReader.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceReader.java
new file mode 100644
index 0000000..63fd328
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceReader.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.reader;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.connector.source.SourceReaderContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
+import org.apache.flink.connector.base.source.reader.SingleThreadMultiplexSourceReaderBase;
+import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
+import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessage;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplitState;
+
+import java.util.Map;
+
+/**
+ * The source reader for the RocketMQ gRPC connector. It emits {@link AckableMessage} records and
+ * does not acknowledge messages itself; acknowledgement is performed by a downstream operator using
+ * the self-contained receipt handle carried by each emitted record. Un-acked messages are
+ * redelivered by the broker after their invisible duration expires, providing the at-least-once
+ * guarantee.
+ */
+@Internal
+public class RocketMQGrpcSourceReader<T>
+ extends SingleThreadMultiplexSourceReaderBase<
+ MessageViewImpl,
+ AckableMessage<T>,
+ RocketMQGrpcSourceSplit,
+ RocketMQGrpcSourceSplitState> {
+
+ public RocketMQGrpcSourceReader(
+ FutureCompletingBlockingQueue<RecordsWithSplitIds<MessageViewImpl>> elementsQueue,
+ RocketMQGrpcSourceFetcherManager fetcherManager,
+ RocketMQGrpcSourceRecordEmitter<T> recordEmitter,
+ Configuration config,
+ SourceReaderContext context) {
+ super(elementsQueue, fetcherManager, recordEmitter, config, context);
+ }
+
+ @Override
+ protected void onSplitFinished(Map<String, RocketMQGrpcSourceSplitState> finishedSplitIds) {
+ // The Pop model is unbounded; splits do not finish.
+ }
+
+ @Override
+ protected RocketMQGrpcSourceSplitState initializedState(RocketMQGrpcSourceSplit split) {
+ return new RocketMQGrpcSourceSplitState(split);
+ }
+
+ @Override
+ protected RocketMQGrpcSourceSplit toSplitType(
+ String splitId, RocketMQGrpcSourceSplitState splitState) {
+ return splitState.toRocketMQGrpcSourceSplit();
+ }
+
+ @VisibleForTesting
+ int getNumAliveFetchers() {
+ return splitFetcherManager.getNumAliveFetchers();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceRecordEmitter.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceRecordEmitter.java
new file mode 100644
index 0000000..2dae65e
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceRecordEmitter.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.reader;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.connector.source.SourceOutput;
+import org.apache.flink.connector.base.source.reader.RecordEmitter;
+import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessage;
+import org.apache.flink.connector.rocketmq.grpc.ack.RocketMQReceiptHandle;
+import org.apache.flink.connector.rocketmq.grpc.ack.RocketMQReceiptHandleCodec;
+import org.apache.flink.connector.rocketmq.grpc.source.deserialization.RocketMQGrpcDeserializationSchema;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplitState;
+import org.apache.flink.util.Collector;
+
+import java.io.IOException;
+
+/**
+ * The {@link RecordEmitter} implementation for the RocketMQ gRPC source. It deserializes the
+ * message body into the value type {@code T}, extracts the credential-free receipt handle from the
+ * SDK message view and emits an {@link AckableMessage} pairing the two so that a downstream
+ * operator can acknowledge the message.
+ */
+@Internal
+public class RocketMQGrpcSourceRecordEmitter<T>
+ implements RecordEmitter<MessageViewImpl, AckableMessage<T>, RocketMQGrpcSourceSplitState> {
+
+ private final RocketMQGrpcDeserializationSchema<T> deserializationSchema;
+ private final String namespace;
+ private final String consumerGroup;
+ private final AckableCollector<T> collector = new AckableCollector<>();
+
+ public RocketMQGrpcSourceRecordEmitter(
+ RocketMQGrpcDeserializationSchema<T> deserializationSchema,
+ String namespace,
+ String consumerGroup) {
+ this.deserializationSchema = deserializationSchema;
+ this.namespace = namespace;
+ this.consumerGroup = consumerGroup;
+ }
+
+ @Override
+ public void emitRecord(
+ MessageViewImpl element,
+ SourceOutput<AckableMessage<T>> output,
+ RocketMQGrpcSourceSplitState splitState)
+ throws IOException {
+ try {
+ final RocketMQReceiptHandle handle =
+ RocketMQReceiptHandleCodec.extract(
+ element.getMessageView(), namespace, consumerGroup);
+ collector.reset(output, element.getEventTime(), handle);
+ deserializationSchema.deserialize(element, collector);
+ splitState.incrementProcessedRecords();
+ } catch (Exception e) {
+ throw new IOException("Failed to deserialize message due to", e);
+ }
+ }
+
+ /** A collector that wraps each deserialized value into an {@link AckableMessage}. */
+ private static class AckableCollector<T> implements Collector<T> {
+
+ private SourceOutput<AckableMessage<T>> sourceOutput;
+ private long timestamp;
+ private RocketMQReceiptHandle handle;
+
+ @Override
+ public void collect(T record) {
+ sourceOutput.collect(new AckableMessage<>(record, handle), timestamp);
+ }
+
+ @Override
+ public void close() {}
+
+ private void reset(
+ SourceOutput<AckableMessage<T>> sourceOutput,
+ long timestamp,
+ RocketMQReceiptHandle handle) {
+ this.sourceOutput = sourceOutput;
+ this.timestamp = timestamp;
+ this.handle = handle;
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceSplitReader.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceSplitReader.java
new file mode 100644
index 0000000..570269a
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/reader/RocketMQGrpcSourceSplitReader.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.reader;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.base.source.reader.RecordsBySplits;
+import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
+import org.apache.flink.connector.base.source.reader.splitreader.SplitReader;
+import org.apache.flink.connector.base.source.reader.splitreader.SplitsChange;
+import org.apache.flink.connector.rocketmq.grpc.source.InvisibleDurationRenewalPolicies;
+import org.apache.flink.connector.rocketmq.grpc.source.InvisibleDurationRenewalPolicy;
+import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit;
+import org.apache.flink.util.FlinkRuntimeException;
+
+import org.apache.rocketmq.client.apis.ClientException;
+import org.apache.rocketmq.client.apis.consumer.LiteSimpleConsumer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/**
+ * The {@link SplitReader} implementation for the RocketMQ gRPC Pop model.
+ *
+ * <p>Because the broker performs message-level load balancing across the consumer group, this
+ * reader does not partition topics. Instead every subtask binds the same main lite topic. To raise
+ * a single subtask's throughput it runs {@code fetch-concurrency} worker threads that share a
+ * single {@link LiteSimpleConsumer}, each issuing a blocking {@code receive()} long poll. Received
+ * messages are handed to the (single) fetcher thread through an internal queue.
+ *
+ * <p>This reader never acknowledges messages: acknowledgement is deferred to a downstream operator
+ * that receives the {@code AckableMessage} records carrying a self-contained receipt handle.
+ * Messages that are never acked become visible again after their invisible duration and are
+ * redelivered by the broker, which provides the at-least-once guarantee.
+ *
+ * <p>When an {@link InvisibleDurationRenewalPolicy} is configured, messages that are still buffered
+ * in the internal queue shortly before they would become visible again are offered to the policy,
+ * which may extend their invisible duration. Messages already handed to the record emitter are
+ * never renewed because renewing refreshes the receipt handle carried downstream.
+ */
+@Internal
+public class RocketMQGrpcSourceSplitReader
+ implements SplitReader<MessageViewImpl, RocketMQGrpcSourceSplit> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RocketMQGrpcSourceSplitReader.class);
+
+ private static final long RECEIVE_BACKOFF_INITIAL_MS = 100L;
+ private static final long RECEIVE_BACKOFF_MAX_MS = 30_000L;
+
+ private final Configuration configuration;
+ private final int fetchConcurrency;
+ private final int maxMessageNum;
+ private final Duration invisibleDuration;
+ private final Duration renewalAheadTime;
+ @Nullable private final InvisibleDurationRenewalPolicy renewalPolicy;
+
+ private final BlockingQueue<MessageViewImpl> elementQueue;
+ private final AtomicBoolean wakeup = new AtomicBoolean(false);
+ private final AtomicBoolean started = new AtomicBoolean(false);
+
+ private volatile boolean closed = false;
+ private ReceiveWorker[] workers;
+ private LiteSimpleConsumer consumer;
+ @Nullable private ScheduledThreadPoolExecutor renewalExecutor;
+
+ public RocketMQGrpcSourceSplitReader(Configuration configuration) {
+ this.configuration = configuration;
+ this.fetchConcurrency = configuration.get(RocketMQGrpcSourceOptions.FETCH_CONCURRENCY);
+ this.maxMessageNum = configuration.get(RocketMQGrpcSourceOptions.MAX_MESSAGE_NUM);
+ this.invisibleDuration = configuration.get(RocketMQGrpcSourceOptions.INVISIBLE_DURATION);
+ this.renewalAheadTime = configuration.get(RocketMQGrpcSourceOptions.RENEWAL_AHEAD_TIME);
+ this.renewalPolicy =
+ InvisibleDurationRenewalPolicies.createFromConfiguration(configuration);
+ if (renewalPolicy != null) {
+ checkArgument(
+ !renewalAheadTime.isNegative() && !renewalAheadTime.isZero(),
+ "renewal-ahead-time must be positive");
+ checkArgument(
+ renewalAheadTime.compareTo(invisibleDuration) < 0,
+ "renewal-ahead-time (%s) must be smaller than the invisible duration (%s)",
+ renewalAheadTime,
+ invisibleDuration);
+ }
+ this.elementQueue = new ArrayBlockingQueue<>(maxMessageNum * fetchConcurrency);
+ }
+
+ @Override
+ public RecordsWithSplitIds<MessageViewImpl> fetch() {
+ if (wakeup.compareAndSet(true, false) || closed) {
+ return new RecordsBySplits.Builder<MessageViewImpl>().build();
+ }
+
+ final MessageViewImpl first;
+ try {
+ first = elementQueue.poll(1, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return new RecordsBySplits.Builder<MessageViewImpl>().build();
+ }
+ if (first == null) {
+ return new RecordsBySplits.Builder<MessageViewImpl>().build();
+ }
+
+ final List<MessageViewImpl> batch = new ArrayList<>(maxMessageNum);
+ batch.add(first);
+ elementQueue.drainTo(batch, maxMessageNum - 1);
+
+ final RecordsBySplits.Builder<MessageViewImpl> builder = new RecordsBySplits.Builder<>();
+ for (MessageViewImpl message : batch) {
+ // From here on the receipt handle travels downstream, so freeze it: waits for an
+ // in-flight renewal to finish and prevents any further renewal.
+ message.markEmitted();
+ builder.add(RocketMQGrpcSourceSplit.SPLIT_ID, message);
+ }
+ return builder.build();
+ }
+
+ @Override
+ public void handleSplitsChanges(SplitsChange<RocketMQGrpcSourceSplit> splitsChanges) {
+ // The placeholder split simply triggers the reader to start consuming.
+ if (!splitsChanges.splits().isEmpty()) {
+ startWorkers();
+ }
+ }
+
+ private void startWorkers() {
+ if (!started.compareAndSet(false, true)) {
+ return;
+ }
+ try {
+ consumer = LiteSimpleConsumerProvider.create(configuration);
+ } catch (ClientException e) {
+ started.set(false);
+ throw new FlinkRuntimeException("Failed to create RocketMQ gRPC LiteSimpleConsumer", e);
+ }
+ if (renewalPolicy != null) {
+ renewalExecutor =
+ new ScheduledThreadPoolExecutor(
+ 1,
+ runnable -> {
+ final Thread thread = new Thread(runnable, "rocketmq-grpc-renewal");
+ thread.setDaemon(true);
+ return thread;
+ });
+ renewalExecutor.setRemoveOnCancelPolicy(true);
+ }
+ workers = new ReceiveWorker[fetchConcurrency];
+ for (int i = 0; i < fetchConcurrency; i++) {
+ final ReceiveWorker worker = new ReceiveWorker(i);
+ workers[i] = worker;
+ worker.start();
+ }
+ LOG.info(
+ "Started {} receive worker(s) sharing one lite consumer bound to topic {}",
+ fetchConcurrency,
+ configuration.get(RocketMQGrpcSourceOptions.MAIN_TOPIC));
+ }
+
+ @Override
+ public void wakeUp() {
+ wakeup.compareAndSet(false, true);
+ }
+
+ @Override
+ public void close() throws Exception {
+ closed = true;
+ if (renewalExecutor != null) {
+ renewalExecutor.shutdownNow();
+ }
+ if (workers != null) {
+ for (ReceiveWorker worker : workers) {
+ worker.shutdown();
+ }
+ }
+ // Close the consumer before joining the workers so that a worker blocked in a receive()
+ // long poll fails fast instead of holding its thread until the await duration elapses.
+ if (consumer != null) {
+ consumer.close();
+ }
+ if (workers != null) {
+ for (ReceiveWorker worker : workers) {
+ worker.join(TimeUnit.SECONDS.toMillis(10));
+ }
+ }
+ }
+
+ private void scheduleRenewal(MessageViewImpl message, int renewalCount, Duration current) {
+ if (renewalExecutor == null || closed) {
+ return;
+ }
+ final long delayMs = Math.max(current.minus(renewalAheadTime).toMillis(), 0L);
+ try {
+ final ScheduledFuture<?> future =
+ renewalExecutor.schedule(
+ () -> renew(message, renewalCount), delayMs, TimeUnit.MILLISECONDS);
+ message.setRenewalFuture(future);
+ } catch (java.util.concurrent.RejectedExecutionException e) {
+ // The reader is closing; the message will simply become visible again.
+ }
+ }
+
+ private void renew(MessageViewImpl message, int renewalCount) {
+ synchronized (message) {
+ if (closed || message.isEmitted()) {
+ return;
+ }
+ final Duration next;
+ try {
+ next = renewalPolicy.renew(message, renewalCount);
+ } catch (Exception e) {
+ LOG.warn(
+ "The renewal policy failed for message {}; it will become visible again",
+ message.getMessageId(),
+ e);
+ return;
+ }
+ if (next == null || next.isZero() || next.isNegative()) {
+ return;
+ }
+ try {
+ consumer.changeInvisibleDuration(message.getMessageView(), next);
+ } catch (ClientException e) {
+ LOG.warn(
+ "Failed to renew the invisible duration of message {}; it will become "
+ + "visible again",
+ message.getMessageId(),
+ e);
+ return;
+ }
+ scheduleRenewal(message, renewalCount + 1, next);
+ }
+ }
+
+ /**
+ * A single receive loop over the shared {@link LiteSimpleConsumer}. Because the consumer is
+ * thread-safe, several workers can call {@code receive()} on it concurrently; a blocking {@code
+ * receive()} only occupies its calling thread, which is why several threads are used to keep
+ * that many long polls in flight.
+ */
+ private class ReceiveWorker extends Thread {
+
+ private final int index;
+
+ private volatile boolean running = true;
+
+ private ReceiveWorker(int index) {
+ super("rocketmq-grpc-receive-worker-" + index);
+ this.index = index;
+ }
+
+ private void shutdown() {
+ running = false;
+ interrupt();
+ }
+
+ @Override
+ public void run() {
+ long backoffMs = RECEIVE_BACKOFF_INITIAL_MS;
+ while (running && !closed) {
+ try {
+ final List<org.apache.rocketmq.client.apis.message.MessageView> messages =
+ consumer.receive(maxMessageNum, invisibleDuration);
+ backoffMs = RECEIVE_BACKOFF_INITIAL_MS;
+ if (!running || closed) {
+ // Dropped messages are redelivered after their invisible duration.
+ break;
+ }
+ for (org.apache.rocketmq.client.apis.message.MessageView message : messages) {
+ final MessageViewImpl view = new MessageViewImpl(message);
+ scheduleRenewal(view, 0, invisibleDuration);
+ elementQueue.put(view);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ } catch (ClientException e) {
+ if (running && !closed) {
+ LOG.warn(
+ "Receive worker {} failed to receive messages, retrying in {} ms",
+ index,
+ backoffMs,
+ e);
+ try {
+ Thread.sleep(backoffMs);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ backoffMs = Math.min(backoffMs * 2, RECEIVE_BACKOFF_MAX_MS);
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplit.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplit.java
new file mode 100644
index 0000000..9ec4d8e
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplit.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.split;
+
+import org.apache.flink.api.connector.source.SourceSplit;
+
+import java.io.Serializable;
+
+/**
+ * A placeholder {@link SourceSplit} for the RocketMQ gRPC Pop consumption model.
+ *
+ * <p>Unlike a pull-based connector, the Pop model performs message-level load balancing on the
+ * broker side: every subtask subscribes to all configured topics and the broker distributes
+ * messages across the consumer group. There is therefore no per-topic, per-queue or offset state to
+ * distribute. This split carries no data; it merely exists because the FLIP-27 {@code Source} API
+ * requires a split type, and assigning a single instance to a reader is what triggers that reader
+ * to start consuming.
+ */
+public class RocketMQGrpcSourceSplit implements SourceSplit, Serializable {
+
+ private static final long serialVersionUID = 2L;
+
+ /** The single, constant split id used by every reader. */
+ public static final String SPLIT_ID = "rocketmq-grpc-pop";
+
+ public static final RocketMQGrpcSourceSplit INSTANCE = new RocketMQGrpcSourceSplit();
+
+ public RocketMQGrpcSourceSplit() {}
+
+ @Override
+ public String splitId() {
+ return SPLIT_ID;
+ }
+
+ @Override
+ public String toString() {
+ return "RocketMQGrpcSourceSplit(" + SPLIT_ID + ")";
+ }
+
+ @Override
+ public int hashCode() {
+ return SPLIT_ID.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof RocketMQGrpcSourceSplit;
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitSerializer.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitSerializer.java
new file mode 100644
index 0000000..01dd709
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitSerializer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.split;
+
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+
+/**
+ * The {@link SimpleVersionedSerializer serializer} for the placeholder {@link
+ * RocketMQGrpcSourceSplit}. The split carries no data, so serialization is empty.
+ */
+public class RocketMQGrpcSourceSplitSerializer
+ implements SimpleVersionedSerializer<RocketMQGrpcSourceSplit> {
+
+ private static final int CURRENT_VERSION = 2;
+
+ @Override
+ public int getVersion() {
+ return CURRENT_VERSION;
+ }
+
+ @Override
+ public byte[] serialize(RocketMQGrpcSourceSplit split) {
+ return new byte[0];
+ }
+
+ @Override
+ public RocketMQGrpcSourceSplit deserialize(int version, byte[] serialized) {
+ return RocketMQGrpcSourceSplit.INSTANCE;
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitState.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitState.java
new file mode 100644
index 0000000..e88ed77
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/source/split/RocketMQGrpcSourceSplitState.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.split;
+
+/**
+ * The mutable runtime state of the placeholder {@link RocketMQGrpcSourceSplit}. The Pop model has
+ * no offset state to track, so this simply counts the number of messages processed, which is useful
+ * for metrics and debugging.
+ */
+public class RocketMQGrpcSourceSplitState {
+
+ private final RocketMQGrpcSourceSplit split;
+ private long processedRecords;
+
+ public RocketMQGrpcSourceSplitState(RocketMQGrpcSourceSplit split) {
+ this.split = split;
+ this.processedRecords = 0L;
+ }
+
+ public long getProcessedRecords() {
+ return processedRecords;
+ }
+
+ public void incrementProcessedRecords() {
+ this.processedRecords++;
+ }
+
+ /** Convert back to an immutable {@link RocketMQGrpcSourceSplit} for checkpointing. */
+ public RocketMQGrpcSourceSplit toRocketMQGrpcSourceSplit() {
+ return split;
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcConnectorOptions.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcConnectorOptions.java
new file mode 100644
index 0000000..03b3d36
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcConnectorOptions.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.table;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ConfigOptions;
+
+import java.time.Duration;
+
+/**
+ * Config options that are used to configure the RocketMQ gRPC SQL connector. These keys are the
+ * user-facing option names in a {@code CREATE TABLE ... WITH (...)} statement.
+ *
+ * <p>Connection/identity options are top-level (mirroring the {@code service-url}/{@code topics}
+ * convention of the Pulsar and Kafka SQL connectors); functional options are grouped by role under
+ * the {@code source.} and {@code sink.} prefixes. The programmatic/SDK-facing options that the
+ * source and sink builders and their runtime consume are defined separately in {@link
+ * org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions}, {@link
+ * org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions} and {@link
+ * org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSinkOptions}.
+ */
+@PublicEvolving
+public class RocketMQGrpcConnectorOptions {
+
+ private RocketMQGrpcConnectorOptions() {}
+
+ // --------------------------------------------------------------------------------------------
+ // Connection options
+ // --------------------------------------------------------------------------------------------
+
+ public static final ConfigOption<String> ENDPOINTS =
+ ConfigOptions.key("endpoints")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The access point (proxy) endpoints the gRPC SDK communicates with, "
+ + "for example '127.0.0.1:8080'.");
+
+ public static final ConfigOption<String> NAMESPACE =
+ ConfigOptions.key("namespace")
+ .stringType()
+ .defaultValue("")
+ .withDescription("The resource namespace of the RocketMQ instance.");
+
+ public static final ConfigOption<String> ACCESS_KEY =
+ ConfigOptions.key("access-key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The access key used for static session credentials.");
+
+ public static final ConfigOption<String> SECRET_KEY =
+ ConfigOptions.key("secret-key")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The secret key used for static session credentials.");
+
+ public static final ConfigOption<String> CREDENTIALS_RESOLVER_CLASS =
+ ConfigOptions.key("credentials-resolver-class")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The fully qualified class name of a CredentialsResolver that resolves "
+ + "the session credentials per endpoint on the TaskManager. "
+ + "When set, it takes precedence over 'access-key' and "
+ + "'secret-key'.");
+
+ public static final ConfigOption<Boolean> TLS_ENABLED =
+ ConfigOptions.key("tls-enabled")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Whether TLS is enabled for the gRPC transport.");
+
+ public static final ConfigOption<Duration> REQUEST_TIMEOUT =
+ ConfigOptions.key("request-timeout")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(3))
+ .withDescription("The request timeout for a single gRPC invocation.");
+
+ public static final ConfigOption<String> TOPIC =
+ ConfigOptions.key("topic")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The default topic to send records to when the table is used as a "
+ + "sink with a value-only serializer.");
+
+ public static final ConfigOption<String> LITE_TOPIC =
+ ConfigOptions.key("sink.lite-topic")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The lite (sub) topic attached to every record when the table is used "
+ + "as a sink. The message is published to the parent topic "
+ + "configured via 'topic' and carries this lite topic, so that "
+ + "a LiteSimpleConsumer bound to the parent topic can receive "
+ + "it. Required for sinks.");
+
+ // --------------------------------------------------------------------------------------------
+ // Source options
+ // --------------------------------------------------------------------------------------------
+
+ public static final ConfigOption<String> CONSUMER_GROUP =
+ ConfigOptions.key("source.consumer-group")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("The consumer group of the SimpleConsumer.");
+
+ public static final ConfigOption<Duration> AWAIT_DURATION =
+ ConfigOptions.key("source.await-duration")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(30))
+ .withDescription(
+ "The long-polling await duration of a single receive() invocation.");
+
+ public static final ConfigOption<Duration> INVISIBLE_DURATION =
+ ConfigOptions.key("source.invisible-duration")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(60))
+ .withDescription(
+ "The invisible duration of a received message. It must be larger than "
+ + "the checkpoint interval plus the checkpoint timeout so that "
+ + "un-acked messages are redelivered after a failure.");
+
+ public static final ConfigOption<Integer> MAX_MESSAGE_NUM =
+ ConfigOptions.key("source.max-message-num")
+ .intType()
+ .defaultValue(32)
+ .withDescription("The maximum number of messages returned by receive().");
+
+ public static final ConfigOption<Integer> FETCH_CONCURRENCY =
+ ConfigOptions.key("source.fetch-concurrency")
+ .intType()
+ .defaultValue(1)
+ .withDescription(
+ "The fetch concurrency of each subtask, i.e. the number of concurrent "
+ + "fetch requests it issues. The requests are served by worker "
+ + "threads sharing a single thread-safe SimpleConsumer, so "
+ + "increasing this raises a single subtask's fetch throughput "
+ + "without changing the Flink parallelism.");
+
+ public static final ConfigOption<String> RENEWAL_POLICY_CLASS =
+ ConfigOptions.key("source.renewal-policy-class")
+ .stringType()
+ .noDefaultValue()
+ .withDescription(
+ "The fully qualified class name of an InvisibleDurationRenewalPolicy "
+ + "implementation consulted shortly before a still-buffered "
+ + "message would become visible again; the policy decides "
+ + "whether to extend its invisible duration. When absent, no "
+ + "renewal is performed.");
+
+ public static final ConfigOption<Duration> RENEWAL_AHEAD_TIME =
+ ConfigOptions.key("source.renewal-ahead-time")
+ .durationType()
+ .defaultValue(Duration.ofSeconds(5))
+ .withDescription(
+ "How long before a buffered message becomes visible again the renewal "
+ + "policy is consulted. Must be positive and smaller than the "
+ + "invisible duration. Only effective when "
+ + "'source.renewal-policy-class' is configured.");
+
+ // --------------------------------------------------------------------------------------------
+ // Sink options
+ // --------------------------------------------------------------------------------------------
+
+ public static final ConfigOption<Integer> MAX_ATTEMPTS =
+ ConfigOptions.key("sink.max-attempts")
+ .intType()
+ .defaultValue(3)
+ .withDescription("The maximum number of send attempts for a message.");
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactory.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactory.java
new file mode 100644
index 0000000..d6968f9
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactory.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.table;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+import org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSinkOptions;
+import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.factories.DeserializationFormatFactory;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.factories.SerializationFormatFactory;
+import org.apache.flink.table.types.DataType;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * The {@link DynamicTableSourceFactory} and {@link DynamicTableSinkFactory} for the RocketMQ gRPC
+ * connector. It is registered under the {@code rocketmq-grpc} identifier.
+ */
+@Internal
+public class RocketMQGrpcDynamicTableFactory
+ implements DynamicTableSourceFactory, DynamicTableSinkFactory {
+
+ public static final String IDENTIFIER = "rocketmq-grpc";
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> requiredOptions() {
+ final Set<ConfigOption<?>> options = new HashSet<>();
+ options.add(RocketMQGrpcConnectorOptions.ENDPOINTS);
+ options.add(RocketMQGrpcConnectorOptions.TOPIC);
+ options.add(FactoryUtil.FORMAT);
+ return options;
+ }
+
+ @Override
+ public Set<ConfigOption<?>> optionalOptions() {
+ final Set<ConfigOption<?>> options = new HashSet<>();
+ options.add(RocketMQGrpcConnectorOptions.NAMESPACE);
+ options.add(RocketMQGrpcConnectorOptions.ACCESS_KEY);
+ options.add(RocketMQGrpcConnectorOptions.SECRET_KEY);
+ options.add(RocketMQGrpcConnectorOptions.CREDENTIALS_RESOLVER_CLASS);
+ options.add(RocketMQGrpcConnectorOptions.TLS_ENABLED);
+ options.add(RocketMQGrpcConnectorOptions.REQUEST_TIMEOUT);
+ options.add(RocketMQGrpcConnectorOptions.LITE_TOPIC);
+ options.add(RocketMQGrpcConnectorOptions.CONSUMER_GROUP);
+ options.add(RocketMQGrpcConnectorOptions.AWAIT_DURATION);
+ options.add(RocketMQGrpcConnectorOptions.INVISIBLE_DURATION);
+ options.add(RocketMQGrpcConnectorOptions.MAX_MESSAGE_NUM);
+ options.add(RocketMQGrpcConnectorOptions.FETCH_CONCURRENCY);
+ options.add(RocketMQGrpcConnectorOptions.RENEWAL_POLICY_CLASS);
+ options.add(RocketMQGrpcConnectorOptions.RENEWAL_AHEAD_TIME);
+ options.add(RocketMQGrpcConnectorOptions.MAX_ATTEMPTS);
+ return options;
+ }
+
+ @Override
+ public DynamicTableSource createDynamicTableSource(Context context) {
+ final FactoryUtil.TableFactoryHelper helper =
+ FactoryUtil.createTableFactoryHelper(this, context);
+ final DecodingFormat<DeserializationSchema<RowData>> decodingFormat =
+ helper.discoverDecodingFormat(
+ DeserializationFormatFactory.class, FactoryUtil.FORMAT);
+ helper.validate();
+
+ final ReadableConfig options = helper.getOptions();
+ checkNotNull(
+ options.get(RocketMQGrpcConnectorOptions.CONSUMER_GROUP),
+ "'%s' is required for the RocketMQ gRPC table source.",
+ RocketMQGrpcConnectorOptions.CONSUMER_GROUP.key());
+
+ final DataType physicalDataType = context.getPhysicalRowDataType();
+ return new RocketMQGrpcDynamicTableSource(
+ buildConfiguration(options),
+ options.get(RocketMQGrpcConnectorOptions.TOPIC),
+ Boundedness.CONTINUOUS_UNBOUNDED,
+ physicalDataType,
+ decodingFormat);
+ }
+
+ @Override
+ public DynamicTableSink createDynamicTableSink(Context context) {
+ final FactoryUtil.TableFactoryHelper helper =
+ FactoryUtil.createTableFactoryHelper(this, context);
+ final EncodingFormat<SerializationSchema<RowData>> encodingFormat =
+ helper.discoverEncodingFormat(SerializationFormatFactory.class, FactoryUtil.FORMAT);
+ helper.validate();
+
+ final ReadableConfig options = helper.getOptions();
+ final String liteTopic = options.get(RocketMQGrpcConnectorOptions.LITE_TOPIC);
+ checkNotNull(
+ liteTopic,
+ "'%s' is required for the RocketMQ gRPC table sink.",
+ RocketMQGrpcConnectorOptions.LITE_TOPIC.key());
+ final DataType physicalDataType = context.getPhysicalRowDataType();
+ return new RocketMQGrpcDynamicTableSink(
+ buildConfiguration(options),
+ options.get(RocketMQGrpcConnectorOptions.TOPIC),
+ liteTopic,
+ physicalDataType,
+ encodingFormat);
+ }
+
+ /**
+ * Translates the user-facing SQL DDL options ({@link RocketMQGrpcConnectorOptions}) into a
+ * {@link Configuration} keyed by the programmatic/SDK options that the source and sink builders
+ * and their runtime read.
+ */
+ private static Configuration buildConfiguration(ReadableConfig options) {
+ final Configuration configuration = new Configuration();
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.ENDPOINTS,
+ RocketMQGrpcOptions.ENDPOINTS);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.NAMESPACE,
+ RocketMQGrpcOptions.NAMESPACE);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.ACCESS_KEY,
+ RocketMQGrpcOptions.ACCESS_KEY);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.SECRET_KEY,
+ RocketMQGrpcOptions.SECRET_KEY);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.CREDENTIALS_RESOLVER_CLASS,
+ RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.TLS_ENABLED,
+ RocketMQGrpcOptions.TLS_ENABLED);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.REQUEST_TIMEOUT,
+ RocketMQGrpcOptions.REQUEST_TIMEOUT);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.TOPIC,
+ RocketMQGrpcSinkOptions.TOPIC);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.LITE_TOPIC,
+ RocketMQGrpcSinkOptions.LITE_TOPIC);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.CONSUMER_GROUP,
+ RocketMQGrpcSourceOptions.CONSUMER_GROUP);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.AWAIT_DURATION,
+ RocketMQGrpcSourceOptions.AWAIT_DURATION);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.INVISIBLE_DURATION,
+ RocketMQGrpcSourceOptions.INVISIBLE_DURATION);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.MAX_MESSAGE_NUM,
+ RocketMQGrpcSourceOptions.MAX_MESSAGE_NUM);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.FETCH_CONCURRENCY,
+ RocketMQGrpcSourceOptions.FETCH_CONCURRENCY);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.RENEWAL_POLICY_CLASS,
+ RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.RENEWAL_AHEAD_TIME,
+ RocketMQGrpcSourceOptions.RENEWAL_AHEAD_TIME);
+ copyMapped(
+ options,
+ configuration,
+ RocketMQGrpcConnectorOptions.MAX_ATTEMPTS,
+ RocketMQGrpcSinkOptions.MAX_ATTEMPTS);
+ return configuration;
+ }
+
+ private static <T> void copyMapped(
+ ReadableConfig source,
+ Configuration target,
+ ConfigOption<T> sourceOption,
+ ConfigOption<T> targetOption) {
+ final T value = source.get(sourceOption);
+ if (value != null) {
+ target.set(targetOption, value);
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSink.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSink.java
new file mode 100644
index 0000000..af35c2e
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSink.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.table;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSink;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.format.EncodingFormat;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.SinkV2Provider;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+
+import java.util.Objects;
+
+/** A {@link DynamicTableSink} for the RocketMQ gRPC connector (at-least-once). */
+@Internal
+public class RocketMQGrpcDynamicTableSink implements DynamicTableSink {
+
+ private final Configuration configuration;
+ private final String topic;
+ private final String liteTopic;
+ private final DataType physicalDataType;
+ private final EncodingFormat<SerializationSchema<RowData>> encodingFormat;
+
+ public RocketMQGrpcDynamicTableSink(
+ Configuration configuration,
+ String topic,
+ String liteTopic,
+ DataType physicalDataType,
+ EncodingFormat<SerializationSchema<RowData>> encodingFormat) {
+ this.configuration = configuration;
+ this.topic = topic;
+ this.liteTopic = liteTopic;
+ this.physicalDataType = physicalDataType;
+ this.encodingFormat = encodingFormat;
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode(ChangelogMode requestedMode) {
+ return ChangelogMode.insertOnly();
+ }
+
+ @Override
+ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
+ final SerializationSchema<RowData> valueSerialization =
+ encodingFormat.createRuntimeEncoder(context, physicalDataType);
+ final RocketMQGrpcRowDataConverter converter =
+ RocketMQGrpcRowDataConverter.forSink(topic, liteTopic, valueSerialization);
+
+ final RocketMQGrpcSink<RowData> sink =
+ RocketMQGrpcSink.<RowData>builder()
+ .setConfig(configuration)
+ .setSerializer(converter)
+ .build();
+
+ return SinkV2Provider.of(sink);
+ }
+
+ @Override
+ public DynamicTableSink copy() {
+ return new RocketMQGrpcDynamicTableSink(
+ configuration, topic, liteTopic, physicalDataType, encodingFormat);
+ }
+
+ @Override
+ public String asSummaryString() {
+ return "RocketMQGrpc";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ final RocketMQGrpcDynamicTableSink that = (RocketMQGrpcDynamicTableSink) o;
+ return Objects.equals(configuration, that.configuration)
+ && Objects.equals(topic, that.topic)
+ && Objects.equals(liteTopic, that.liteTopic)
+ && Objects.equals(physicalDataType, that.physicalDataType)
+ && Objects.equals(encodingFormat, that.encodingFormat);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(configuration, topic, liteTopic, physicalDataType, encodingFormat);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSource.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSource.java
new file mode 100644
index 0000000..5cb7f54
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableSource.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.table;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.connector.source.Boundedness;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.ack.AckableMessage;
+import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSource;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.ProviderContext;
+import org.apache.flink.table.connector.format.DecodingFormat;
+import org.apache.flink.table.connector.source.DataStreamScanProvider;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.connector.source.ScanTableSource;
+import org.apache.flink.table.connector.source.abilities.SupportsReadingMetadata;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.DataType;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/** A {@link ScanTableSource} for the RocketMQ gRPC connector. */
+@Internal
+public class RocketMQGrpcDynamicTableSource implements ScanTableSource, SupportsReadingMetadata {
+
+ private final Configuration configuration;
+ private final String topic;
+ private final Boundedness boundedness;
+ private final DataType physicalDataType;
+ private final DecodingFormat<DeserializationSchema<RowData>> decodingFormat;
+
+ /** The metadata keys applied by {@link #applyReadableMetadata(List, DataType)}. */
+ private List<String> metadataKeys;
+
+ /** The produced data type including the appended metadata columns. */
+ private DataType producedDataType;
+
+ public RocketMQGrpcDynamicTableSource(
+ Configuration configuration,
+ String topic,
+ Boundedness boundedness,
+ DataType physicalDataType,
+ DecodingFormat<DeserializationSchema<RowData>> decodingFormat) {
+ this.configuration = configuration;
+ this.topic = topic;
+ this.boundedness = boundedness;
+ this.physicalDataType = physicalDataType;
+ this.decodingFormat = decodingFormat;
+ this.metadataKeys = Collections.emptyList();
+ this.producedDataType = physicalDataType;
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode() {
+ return decodingFormat.getChangelogMode();
+ }
+
+ @Override
+ public Map<String, DataType> listReadableMetadata() {
+ final Map<String, DataType> metadataMap = new LinkedHashMap<>();
+ Stream.of(RocketMQGrpcReadableMetadata.values())
+ .forEach(metadata -> metadataMap.put(metadata.getKey(), metadata.getDataType()));
+ return metadataMap;
+ }
+
+ @Override
+ public void applyReadableMetadata(List<String> metadataKeys, DataType producedDataType) {
+ this.metadataKeys = metadataKeys;
+ this.producedDataType = producedDataType;
+ }
+
+ @Override
+ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext context) {
+ final DeserializationSchema<RowData> valueDeserialization =
+ decodingFormat.createRuntimeDecoder(context, physicalDataType);
+ final TypeInformation<RowData> producedType =
+ context.createTypeInformation(producedDataType);
+ final List<RocketMQGrpcReadableMetadata.MetadataConverter> metadataConverters =
+ metadataKeys.stream()
+ .map(
+ key ->
+ Stream.of(RocketMQGrpcReadableMetadata.values())
+ .filter(metadata -> metadata.getKey().equals(key))
+ .findFirst()
+ .orElseThrow(IllegalStateException::new)
+ .getConverter())
+ .collect(Collectors.toList());
+ final RocketMQGrpcRowDataConverter converter =
+ RocketMQGrpcRowDataConverter.forSource(
+ valueDeserialization, metadataConverters, producedType);
+
+ final RocketMQGrpcSource<RowData> source =
+ RocketMQGrpcSource.<RowData>builder()
+ .setConfig(configuration)
+ .setMainTopic(topic)
+ .setBoundedness(boundedness)
+ .setDeserializer(converter)
+ .build();
+
+ // The source produces AckableMessage<RowData>; the SQL/Table path does not perform
+ // downstream acknowledgement, so the receipt handle is stripped here and only the RowData
+ // value is forwarded.
+ return new DataStreamScanProvider() {
+ @Override
+ public DataStream<RowData> produceDataStream(
+ ProviderContext providerContext, StreamExecutionEnvironment execEnv) {
+ final DataStreamSource<AckableMessage<RowData>> sourceStream =
+ execEnv.fromSource(
+ source, WatermarkStrategy.noWatermarks(), "RocketMQGrpcSource");
+ final SingleOutputStreamOperator<RowData> valueStream =
+ sourceStream.map(AckableMessage::getValue).returns(producedType);
+ providerContext.generateUid("rocketmq-grpc-source").ifPresent(valueStream::uid);
+ return valueStream;
+ }
+
+ @Override
+ public boolean isBounded() {
+ return Boundedness.BOUNDED == boundedness;
+ }
+ };
+ }
+
+ @Override
+ public DynamicTableSource copy() {
+ final RocketMQGrpcDynamicTableSource copy =
+ new RocketMQGrpcDynamicTableSource(
+ configuration, topic, boundedness, physicalDataType, decodingFormat);
+ copy.metadataKeys = new ArrayList<>(metadataKeys);
+ copy.producedDataType = producedDataType;
+ return copy;
+ }
+
+ @Override
+ public String asSummaryString() {
+ return "RocketMQGrpc";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ final RocketMQGrpcDynamicTableSource that = (RocketMQGrpcDynamicTableSource) o;
+ return Objects.equals(configuration, that.configuration)
+ && Objects.equals(topic, that.topic)
+ && boundedness == that.boundedness
+ && Objects.equals(physicalDataType, that.physicalDataType)
+ && Objects.equals(decodingFormat, that.decodingFormat)
+ && Objects.equals(metadataKeys, that.metadataKeys)
+ && Objects.equals(producedDataType, that.producedDataType);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ configuration,
+ topic,
+ boundedness,
+ physicalDataType,
+ decodingFormat,
+ metadataKeys,
+ producedDataType);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcReadableMetadata.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcReadableMetadata.java
new file mode 100644
index 0000000..a2019c7
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcReadableMetadata.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.table;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericArrayData;
+import org.apache.flink.table.data.GenericMapData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.types.DataType;
+
+import javax.annotation.Nullable;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * The readable metadata of a RocketMQ gRPC message that a SQL table source can expose as {@code
+ * METADATA VIRTUAL} columns.
+ */
+@Internal
+public enum RocketMQGrpcReadableMetadata {
+ MESSAGE_ID(
+ "message_id",
+ DataTypes.STRING().nullable(),
+ messageView -> StringData.fromString(messageView.getMessageId())),
+
+ TOPIC(
+ "topic",
+ DataTypes.STRING().nullable(),
+ messageView -> StringData.fromString(messageView.getTopic())),
+
+ TAG(
+ "tag",
+ DataTypes.STRING().nullable(),
+ messageView ->
+ messageView.getTag() == null
+ ? null
+ : StringData.fromString(messageView.getTag())),
+
+ KEYS(
+ "keys",
+ DataTypes.ARRAY(DataTypes.STRING()).nullable(),
+ messageView ->
+ new GenericArrayData(
+ messageView.getKeys().stream().map(StringData::fromString).toArray())),
+
+ DELIVERY_ATTEMPT(
+ "delivery_attempt", DataTypes.INT().nullable(), MessageView::getDeliveryAttempt),
+
+ BORN_TIMESTAMP(
+ "born_timestamp",
+ DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3).nullable(),
+ messageView -> TimestampData.fromEpochMillis(messageView.getEventTime())),
+
+ PROPERTIES(
+ "properties",
+ DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()).nullable(),
+ messageView -> {
+ final Map<StringData, StringData> map = new HashMap<>();
+ messageView
+ .getProperties()
+ .forEach(
+ (key, value) ->
+ map.put(
+ StringData.fromString(key),
+ StringData.fromString(value)));
+ return new GenericMapData(map);
+ });
+
+ private final String key;
+ private final DataType dataType;
+ private final MetadataConverter converter;
+
+ RocketMQGrpcReadableMetadata(String key, DataType dataType, MetadataConverter converter) {
+ this.key = key;
+ this.dataType = dataType;
+ this.converter = converter;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ public MetadataConverter getConverter() {
+ return converter;
+ }
+
+ /** Converts a message attribute into the internal data structure of the metadata column. */
+ @FunctionalInterface
+ public interface MetadataConverter extends Serializable {
+ @Nullable
+ Object read(MessageView messageView);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverter.java b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverter.java
new file mode 100644
index 0000000..5ce50a4
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverter.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.table;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.serialization.SerializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.connector.rocketmq.grpc.sink.serialization.RocketMQGrpcSerializationSchema;
+import org.apache.flink.connector.rocketmq.grpc.source.deserialization.RocketMQGrpcDeserializationSchema;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.util.Collector;
+
+import org.apache.rocketmq.client.apis.message.Message;
+import org.apache.rocketmq.client.apis.message.MessageBuilder;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/**
+ * Converts between RocketMQ gRPC {@link Message}/{@link MessageView} and Flink {@link RowData}. The
+ * value part of the record is (de)serialized with the format-provided value schema; the message
+ * body carries the encoded value. On the source side, requested metadata columns are appended after
+ * the physical columns produced by the value format.
+ */
+@Internal
+public class RocketMQGrpcRowDataConverter
+ implements RocketMQGrpcDeserializationSchema<RowData>,
+ RocketMQGrpcSerializationSchema<RowData> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Nullable private final String topic;
+ @Nullable private final String liteTopic;
+ @Nullable private final DeserializationSchema<RowData> valueDeserialization;
+ @Nullable private final SerializationSchema<RowData> valueSerialization;
+ private final List<RocketMQGrpcReadableMetadata.MetadataConverter> metadataConverters;
+ private final TypeInformation<RowData> producedType;
+
+ private transient MetadataAppendingCollector metadataAppendingCollector;
+
+ private RocketMQGrpcRowDataConverter(
+ @Nullable String topic,
+ @Nullable String liteTopic,
+ @Nullable DeserializationSchema<RowData> valueDeserialization,
+ @Nullable SerializationSchema<RowData> valueSerialization,
+ List<RocketMQGrpcReadableMetadata.MetadataConverter> metadataConverters,
+ @Nullable TypeInformation<RowData> producedType) {
+ this.topic = topic;
+ this.liteTopic = liteTopic;
+ this.valueDeserialization = valueDeserialization;
+ this.valueSerialization = valueSerialization;
+ this.metadataConverters = metadataConverters;
+ this.producedType = producedType;
+ }
+
+ /** Create a converter used by the table source to decode a {@link MessageView}. */
+ public static RocketMQGrpcRowDataConverter forSource(
+ DeserializationSchema<RowData> valueDeserialization,
+ List<RocketMQGrpcReadableMetadata.MetadataConverter> metadataConverters,
+ TypeInformation<RowData> producedType) {
+ return new RocketMQGrpcRowDataConverter(
+ null,
+ null,
+ checkNotNull(valueDeserialization),
+ null,
+ checkNotNull(metadataConverters),
+ checkNotNull(producedType));
+ }
+
+ /** Create a converter used by the table sink to encode a {@link RowData}. */
+ public static RocketMQGrpcRowDataConverter forSink(
+ String topic, String liteTopic, SerializationSchema<RowData> valueSerialization) {
+ return new RocketMQGrpcRowDataConverter(
+ checkNotNull(topic),
+ checkNotNull(liteTopic),
+ null,
+ checkNotNull(valueSerialization),
+ Collections.emptyList(),
+ null);
+ }
+
+ @Override
+ public void open(DeserializationSchema.InitializationContext context) throws Exception {
+ checkNotNull(valueDeserialization).open(context);
+ this.metadataAppendingCollector = new MetadataAppendingCollector(metadataConverters);
+ }
+
+ @Override
+ public void open(SerializationSchema.InitializationContext context) throws Exception {
+ checkNotNull(valueSerialization).open(context);
+ }
+
+ @Override
+ public void deserialize(MessageView messageView, Collector<RowData> out) throws IOException {
+ if (metadataConverters.isEmpty()) {
+ checkNotNull(valueDeserialization).deserialize(messageView.getBody(), out);
+ return;
+ }
+ metadataAppendingCollector.reset(messageView, out);
+ checkNotNull(valueDeserialization)
+ .deserialize(messageView.getBody(), metadataAppendingCollector);
+ }
+
+ @Override
+ public Message serialize(RowData element, MessageBuilder messageBuilder, Long timestamp) {
+ final byte[] body = checkNotNull(valueSerialization).serialize(element);
+ return messageBuilder.setTopic(topic).setLiteTopic(liteTopic).setBody(body).build();
+ }
+
+ @Override
+ public TypeInformation<RowData> getProducedType() {
+ return checkNotNull(producedType);
+ }
+
+ /**
+ * Appends the requested metadata columns after the physical columns of each row produced by the
+ * value format. Like other connectors' metadata support, it relies on the format's runtime
+ * decoder producing {@link GenericRowData}.
+ */
+ private static final class MetadataAppendingCollector implements Collector<RowData> {
+
+ private final List<RocketMQGrpcReadableMetadata.MetadataConverter> metadataConverters;
+
+ private MessageView messageView;
+ private Collector<RowData> out;
+
+ private MetadataAppendingCollector(
+ List<RocketMQGrpcReadableMetadata.MetadataConverter> metadataConverters) {
+ this.metadataConverters = metadataConverters;
+ }
+
+ private void reset(MessageView messageView, Collector<RowData> out) {
+ this.messageView = messageView;
+ this.out = out;
+ }
+
+ @Override
+ public void collect(RowData physicalRow) {
+ final GenericRowData physical = (GenericRowData) physicalRow;
+ final int physicalArity = physical.getArity();
+ final GenericRowData produced =
+ new GenericRowData(
+ physical.getRowKind(), physicalArity + metadataConverters.size());
+ for (int pos = 0; pos < physicalArity; pos++) {
+ produced.setField(pos, physical.getField(pos));
+ }
+ for (int pos = 0; pos < metadataConverters.size(); pos++) {
+ produced.setField(
+ physicalArity + pos, metadataConverters.get(pos).read(messageView));
+ }
+ out.collect(produced);
+ }
+
+ @Override
+ public void close() {}
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector-rocketmq-grpc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
new file mode 100644
index 0000000..7e42fa2
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.connector.rocketmq.grpc.table.RocketMQGrpcDynamicTableFactory
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcITCase.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcITCase.java
new file mode 100644
index 0000000..9ce6d06
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcITCase.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc;
+
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.ack.RocketMQThrottleProcessFunction;
+import org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSink;
+import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSource;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.test.util.AbstractTestBase;
+
+import org.apache.rocketmq.client.apis.ClientConfiguration;
+import org.apache.rocketmq.client.apis.ClientServiceProvider;
+import org.apache.rocketmq.client.apis.StaticSessionCredentialsProvider;
+import org.apache.rocketmq.client.apis.message.Message;
+import org.apache.rocketmq.client.apis.producer.Producer;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Integration tests for the RocketMQ gRPC connector. Requires a running RocketMQ 5.x instance with
+ * gRPC proxy enabled. Connection details are read from environment variables, so this test is
+ * {@link Disabled} by default and must be run manually after exporting {@code
+ * ROCKETMQ_GRPC_ENDPOINTS}, {@code ROCKETMQ_GRPC_ACCESS_KEY} and {@code ROCKETMQ_GRPC_SECRET_KEY}.
+ */
+@Disabled(
+ "Requires a running RocketMQ 5.x instance with gRPC proxy. Set the ROCKETMQ_GRPC_* "
+ + "environment variables and run manually.")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class RocketMQGrpcITCase extends AbstractTestBase {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RocketMQGrpcITCase.class);
+
+ // -- Instance configuration (read from environment; never hard-code credentials) --
+ private static final String ENDPOINTS = System.getenv("ROCKETMQ_GRPC_ENDPOINTS");
+ private static final String ACCESS_KEY = System.getenv("ROCKETMQ_GRPC_ACCESS_KEY");
+ private static final String SECRET_KEY = System.getenv("ROCKETMQ_GRPC_SECRET_KEY");
+
+ // -- Topics and groups --
+ private static final String SINK_TOPIC = "flink-sink-1";
+ private static final String SINK_LITE_TOPIC = "flink-sink-1-lite";
+ private static final String SOURCE_TOPIC = "flink-source";
+ private static final String E2E_SOURCE_TOPIC = "flink-source-1";
+ private static final String E2E_SINK_TOPIC = "flink-sink-2";
+ private static final String E2E_SINK_LITE_TOPIC = "flink-sink-2-lite";
+ private static final String CONSUMER_GROUP = "GID-flink";
+
+ private static final int NUM_MESSAGES = 50;
+ private static final long COLLECT_TIMEOUT_SECONDS = 120;
+
+ /**
+ * Each test run uses a unique prefix so that messages from previous runs can be distinguished
+ * from the current run's messages.
+ */
+ private final String runId = UUID.randomUUID().toString().substring(0, 8);
+
+ // -----------------------------------------------------------------------
+ // Test 1: Sink writes messages
+ // -----------------------------------------------------------------------
+
+ @Test
+ @Order(1)
+ void testSinkWritesMessages() throws Exception {
+ final String prefix = "sink-" + runId + "-";
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ final List<String> messages = new ArrayList<>(NUM_MESSAGES);
+ for (int i = 0; i < NUM_MESSAGES; i++) {
+ messages.add(prefix + i);
+ }
+
+ final RocketMQGrpcSink<String> sink =
+ RocketMQGrpcSink.<String>builder()
+ .setEndpoints(ENDPOINTS)
+ .setTopic(SINK_TOPIC)
+ .setLiteTopic(SINK_LITE_TOPIC)
+ .setConfig(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY)
+ .setConfig(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY)
+ .setValueOnlySerializer(new SimpleStringSchema())
+ .build();
+
+ env.fromData(messages).sinkTo(sink);
+ env.execute("RocketMQ gRPC Sink Test");
+ LOG.info("Successfully wrote {} messages to topic {}", NUM_MESSAGES, SINK_TOPIC);
+ }
+
+ // -----------------------------------------------------------------------
+ // Test 2: Source reads messages
+ // -----------------------------------------------------------------------
+
+ @Test
+ @Order(2)
+ void testSourceReadsMessages() throws Exception {
+ final String prefix = "src-" + runId + "-";
+
+ // Seed messages using the RocketMQ Producer API directly.
+ seedMessagesViaProducer(SOURCE_TOPIC, NUM_MESSAGES, prefix);
+
+ // Read them back using the Flink source.
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ final RocketMQGrpcSource<String> source =
+ RocketMQGrpcSource.<String>builder()
+ .setEndpoints(ENDPOINTS)
+ .setConsumerGroup(CONSUMER_GROUP)
+ .setMainTopic(SOURCE_TOPIC)
+ .setConfig(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY)
+ .setConfig(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY)
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build();
+
+ final DataStream<String> stream =
+ env.fromSource(source, WatermarkStrategy.noWatermarks(), "RocketMQ gRPC Source")
+ .process(
+ new RocketMQThrottleProcessFunction<String>(
+ ackConfiguration(), value -> Optional.empty()))
+ .returns(String.class);
+ stream.addSink(new CollectingSinkFunction());
+
+ // Collect messages, filtering by the current run's prefix.
+ final List<String> results =
+ CollectingSinkFunction.startAndCollect(env, NUM_MESSAGES, prefix);
+ LOG.info("Collected {} messages (prefix={}) from source", results.size(), prefix);
+
+ assertThat(results).hasSize(NUM_MESSAGES);
+ for (int i = 0; i < NUM_MESSAGES; i++) {
+ assertThat(results).contains(prefix + i);
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // Test 3: End-to-end pipeline (Source -> map -> Sink)
+ // -----------------------------------------------------------------------
+
+ @Test
+ @Order(3)
+ void testEndToEndPipeline() throws Exception {
+ final String prefix = "e2e-" + runId + "-";
+
+ // Seed messages into the e2e source topic.
+ seedMessagesViaProducer(E2E_SOURCE_TOPIC, NUM_MESSAGES, prefix);
+
+ // Build pipeline: Source reads from flink-source-1 -> uppercase -> Sink to flink-sink-2.
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ final RocketMQGrpcSource<String> source =
+ RocketMQGrpcSource.<String>builder()
+ .setEndpoints(ENDPOINTS)
+ .setConsumerGroup(CONSUMER_GROUP)
+ .setMainTopic(E2E_SOURCE_TOPIC)
+ .setConfig(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY)
+ .setConfig(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY)
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build();
+
+ final RocketMQGrpcSink<String> sink =
+ RocketMQGrpcSink.<String>builder()
+ .setEndpoints(ENDPOINTS)
+ .setTopic(E2E_SINK_TOPIC)
+ .setLiteTopic(E2E_SINK_LITE_TOPIC)
+ .setConfig(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY)
+ .setConfig(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY)
+ .setValueOnlySerializer(new SimpleStringSchema())
+ .build();
+
+ final DataStream<String> stream =
+ env.fromSource(source, WatermarkStrategy.noWatermarks(), "RocketMQ gRPC Source")
+ .process(
+ new RocketMQThrottleProcessFunction<String>(
+ ackConfiguration(), value -> Optional.empty()))
+ .returns(String.class);
+ stream.map(String::toUpperCase).sinkTo(sink);
+
+ // Run the pipeline in a background thread.
+ final Thread jobThread =
+ new Thread(
+ () -> {
+ try {
+ env.execute("E2E Pipeline");
+ } catch (Exception e) {
+ LOG.info("E2E pipeline ended: {}", e.getMessage());
+ }
+ });
+ jobThread.setDaemon(true);
+ jobThread.start();
+
+ // Wait for the pipeline to process.
+ Thread.sleep(30_000);
+
+ env.close();
+ jobThread.interrupt();
+ jobThread.join(10_000);
+
+ // Verify by reading from the sink topic, filtering by the uppercased prefix.
+ final String upperPrefix = prefix.toUpperCase();
+ final List<String> results =
+ readMessagesViaConsumer(E2E_SINK_TOPIC, NUM_MESSAGES, upperPrefix);
+ LOG.info(
+ "Read {} messages from sink topic {} (prefix={})",
+ results.size(),
+ E2E_SINK_TOPIC,
+ upperPrefix);
+
+ assertThat(results).hasSize(NUM_MESSAGES);
+ for (int i = 0; i < NUM_MESSAGES; i++) {
+ assertThat(results).contains(upperPrefix + i);
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // Helpers
+ // -----------------------------------------------------------------------
+
+ /**
+ * Build a client {@link Configuration} for the downstream ack operators; credentials come from
+ * the environment and never travel in the data stream.
+ */
+ private Configuration ackConfiguration() {
+ final Configuration config = new Configuration();
+ config.set(RocketMQGrpcOptions.ENDPOINTS, ENDPOINTS);
+ config.set(RocketMQGrpcOptions.ACCESS_KEY, ACCESS_KEY);
+ config.set(RocketMQGrpcOptions.SECRET_KEY, SECRET_KEY);
+ return config;
+ }
+
+ /** Seed messages into the given topic using the RocketMQ gRPC Producer API directly. */
+ private void seedMessagesViaProducer(String topic, int count, String prefix) throws Exception {
+ final ClientServiceProvider provider = ClientServiceProvider.loadService();
+ final ClientConfiguration clientConfig =
+ ClientConfiguration.newBuilder()
+ .setEndpoints(ENDPOINTS)
+ .setCredentialProvider(
+ new StaticSessionCredentialsProvider(ACCESS_KEY, SECRET_KEY))
+ .build();
+
+ try (Producer producer =
+ provider.newProducerBuilder()
+ .setClientConfiguration(clientConfig)
+ .setTopics(topic)
+ .build()) {
+ for (int i = 0; i < count; i++) {
+ final Message message =
+ provider.newMessageBuilder()
+ .setTopic(topic)
+ .setBody((prefix + i).getBytes(StandardCharsets.UTF_8))
+ .build();
+ producer.send(message);
+ }
+ }
+ LOG.info("Seeded {} messages into topic {} (prefix={})", count, topic, prefix);
+
+ // Give the broker a moment to make messages available for consumers.
+ Thread.sleep(3000);
+ }
+
+ /**
+ * Read messages from a topic using the RocketMQ gRPC SimpleConsumer API directly, filtering by
+ * the given prefix. Used to verify end-to-end pipeline output.
+ */
+ private List<String> readMessagesViaConsumer(String topic, int expectedCount, String prefix)
+ throws Exception {
+ final ClientServiceProvider provider = ClientServiceProvider.loadService();
+ final ClientConfiguration clientConfig =
+ ClientConfiguration.newBuilder()
+ .setEndpoints(ENDPOINTS)
+ .setCredentialProvider(
+ new StaticSessionCredentialsProvider(ACCESS_KEY, SECRET_KEY))
+ .build();
+
+ final List<String> results = new ArrayList<>();
+ final org.apache.rocketmq.client.apis.consumer.FilterExpression filterExpression =
+ new org.apache.rocketmq.client.apis.consumer.FilterExpression("*");
+ final java.util.Map<String, org.apache.rocketmq.client.apis.consumer.FilterExpression>
+ subscriptions = Collections.singletonMap(topic, filterExpression);
+
+ try (org.apache.rocketmq.client.apis.consumer.SimpleConsumer consumer =
+ provider.newSimpleConsumerBuilder()
+ .setClientConfiguration(clientConfig)
+ .setConsumerGroup(CONSUMER_GROUP)
+ .setAwaitDuration(Duration.ofSeconds(15))
+ .setSubscriptionExpressions(subscriptions)
+ .build()) {
+
+ final long deadline = System.currentTimeMillis() + COLLECT_TIMEOUT_SECONDS * 1000;
+ while (results.size() < expectedCount && System.currentTimeMillis() < deadline) {
+ final List<org.apache.rocketmq.client.apis.message.MessageView> messages =
+ consumer.receive(32, Duration.ofSeconds(15));
+ for (org.apache.rocketmq.client.apis.message.MessageView msg : messages) {
+ final ByteBuffer buf = msg.getBody();
+ final byte[] bytes = new byte[buf.remaining()];
+ buf.get(bytes);
+ final String body = new String(bytes, StandardCharsets.UTF_8);
+ consumer.ack(msg);
+ if (body.startsWith(prefix)) {
+ results.add(body);
+ }
+ }
+ }
+ }
+ return results;
+ }
+
+ /**
+ * A collecting sink function that gathers emitted records into a thread-safe queue. Used to
+ * retrieve results from an unbounded source by cancelling the job once enough records matching
+ * the prefix are collected.
+ */
+ @SuppressWarnings("deprecation")
+ private static class CollectingSinkFunction
+ extends org.apache.flink.streaming.api.functions.sink.RichSinkFunction<String> {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final LinkedBlockingQueue<String> QUEUE = new LinkedBlockingQueue<>();
+
+ @Override
+ public void invoke(String value, Context context) {
+ QUEUE.add(value);
+ }
+
+ /**
+ * Execute the given environment and collect at least {@code expectedCount} records matching
+ * the prefix. The job is cancelled once enough records are gathered or after a timeout.
+ */
+ static List<String> startAndCollect(
+ StreamExecutionEnvironment env, int expectedCount, String prefix) throws Exception {
+ QUEUE.clear();
+
+ final Thread jobThread =
+ new Thread(
+ () -> {
+ try {
+ env.execute("Collecting Job");
+ } catch (Exception e) {
+ LOG.info("Collecting job ended: {}", e.getMessage());
+ }
+ });
+ jobThread.setDaemon(true);
+ jobThread.start();
+
+ // Collect messages matching the prefix until we have enough or timeout.
+ final List<String> results = new ArrayList<>();
+ final long deadline = System.currentTimeMillis() + COLLECT_TIMEOUT_SECONDS * 1000;
+ while (results.size() < expectedCount && System.currentTimeMillis() < deadline) {
+ final String item = QUEUE.poll(500, TimeUnit.MILLISECONDS);
+ if (item != null && item.startsWith(prefix)) {
+ results.add(item);
+ }
+ }
+
+ env.close();
+ jobThread.interrupt();
+ jobThread.join(10_000);
+
+ // Drain remaining matching items.
+ final List<String> remaining = new ArrayList<>();
+ QUEUE.drainTo(remaining);
+ for (String item : remaining) {
+ if (item.startsWith(prefix)) {
+ results.add(item);
+ }
+ }
+ return results;
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcLiteE2EVerify.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcLiteE2EVerify.java
new file mode 100644
index 0000000..bea6673
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/RocketMQGrpcLiteE2EVerify.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc;
+
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.ack.RocketMQThrottleProcessFunction;
+import org.apache.flink.connector.rocketmq.grpc.sink.RocketMQGrpcSink;
+import org.apache.flink.connector.rocketmq.grpc.source.InvisibleDurationRenewalPolicy;
+import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSource;
+import org.apache.flink.connector.rocketmq.grpc.source.RocketMQGrpcSourceOptions;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Manual end-to-end verification of the RocketMQ gRPC connector against a live cluster (public
+ * proxy SLB). Run with {@code mvn test-compile exec:java}. Phases:
+ *
+ * <ol>
+ * <li>Sink: publish lite messages to the parent topic.
+ * <li>Source + throttle: consume them; the throttle policy defers selected messages once by 10s
+ * (changeInvisibleDuration), verifying the throttle really delays re-delivery.
+ * <li>Renewal: a slow pipeline keeps messages queued in the split reader beyond the renewal
+ * trigger point, verifying the invisible-duration renewal fires and prevents duplicates.
+ * </ol>
+ */
+public class RocketMQGrpcLiteE2EVerify {
+
+ private static final String ENDPOINTS =
+ System.getProperty("rocketmq.endpoints", "47.98.110.140:8081");
+ private static final String MAIN_TOPIC = "LiteTest";
+ private static final String CONSUMER_GROUP = "LiteSimpleConsumerGroup";
+
+ private static final String RUN_ID = UUID.randomUUID().toString().substring(0, 8);
+
+ // value -> arrival timestamps (ms)
+ private static final Map<String, List<Long>> ARRIVALS = new ConcurrentHashMap<>();
+ // values the throttle policy has already deferred once
+ private static final Map<String, Boolean> THROTTLED_ONCE = new ConcurrentHashMap<>();
+ // renewal invocations observed by the renewal policy
+ private static final AtomicInteger RENEWALS = new AtomicInteger();
+
+ public static void main(String[] args) {
+ // Run each phase in its own JVM: a finished local job cannot be cancelled cleanly, and a
+ // lingering same-group consumer would steal the next phase's messages.
+ final String phase = args.length > 0 ? args[0] : "phase1";
+ System.out.println("=== RocketMQGrpcLiteE2EVerify " + phase + " runId=" + RUN_ID + " ===");
+ try {
+ if ("phase2".equals(phase)) {
+ phase2Renewal();
+ } else {
+ phase1SinkAndThrottle();
+ }
+ System.out.println("=== " + phase + " PASSED ===");
+ System.exit(0);
+ } catch (Throwable t) {
+ t.printStackTrace();
+ System.exit(1);
+ }
+ }
+
+ // ------------------------------------------------------------------
+ // Phase 1: sink writes; source + throttle policy verification
+ // ------------------------------------------------------------------
+
+ private static void phase1SinkAndThrottle() throws Exception {
+ final String normalPrefix = "n-" + RUN_ID + "-";
+ final String throttlePrefix = "thr-" + RUN_ID + "-";
+ final List<String> messages = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ messages.add(normalPrefix + i);
+ }
+ for (int i = 0; i < 5; i++) {
+ messages.add(throttlePrefix + i);
+ }
+ runSinkJob(messages, "flink-e2e-lite");
+ System.out.println("[phase1] sink wrote " + messages.size() + " lite messages");
+
+ final StreamExecutionEnvironment env = localEnv();
+ final RocketMQGrpcSource<String> source =
+ RocketMQGrpcSource.<String>builder()
+ .setEndpoints(ENDPOINTS)
+ .setConsumerGroup(CONSUMER_GROUP)
+ .setMainTopic(MAIN_TOPIC)
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build();
+
+ final Configuration ackConfig = new Configuration();
+ ackConfig.set(RocketMQGrpcOptions.ENDPOINTS, ENDPOINTS);
+
+ final String tp = throttlePrefix;
+ final DataStream<String> stream =
+ env.fromSource(source, WatermarkStrategy.noWatermarks(), "grpc-source")
+ .process(
+ new RocketMQThrottleProcessFunction<String>(
+ ackConfig,
+ value -> {
+ if (value.startsWith(tp)
+ && THROTTLED_ONCE.putIfAbsent(value, true)
+ == null) {
+ return Optional.of(Duration.ofSeconds(10));
+ }
+ return Optional.empty();
+ }))
+ .returns(String.class);
+ stream.map(RocketMQGrpcLiteE2EVerify::record);
+
+ ARRIVALS.clear();
+ runJobUntil(
+ env,
+ 180,
+ () -> {
+ for (int i = 0; i < 10; i++) {
+ if (count(normalPrefix + i) < 1) {
+ return false;
+ }
+ }
+ for (int i = 0; i < 5; i++) {
+ if (count(tp + i) < 2) {
+ return false;
+ }
+ }
+ return true;
+ });
+
+ for (int i = 0; i < 5; i++) {
+ final List<Long> times = ARRIVALS.get(tp + i);
+ check(times != null && times.size() >= 2, "throttled message not redelivered: " + i);
+ final long gapMs = times.get(1) - times.get(0);
+ check(gapMs >= 8000, "throttle gap too small: " + gapMs + "ms for " + tp + i);
+ System.out.println("[phase1] " + tp + i + " redelivered after " + gapMs + " ms");
+ }
+ System.out.println("[phase1] PASS: send/receive + throttle (defer via invisible duration)");
+ }
+
+ // ------------------------------------------------------------------
+ // Phase 2: invisible-duration renewal under backpressure
+ // ------------------------------------------------------------------
+
+ private static void phase2Renewal() throws Exception {
+ final String prefix = "rnw-" + RUN_ID + "-";
+ final int total = 60;
+ // Spread across lite topics: the broker paces event dispatch per lite topic when un-acked
+ // messages accumulate, which would starve the backpressure scenario otherwise.
+ final StreamExecutionEnvironment sinkEnv = localEnv();
+ for (int t = 0; t < 20; t++) {
+ final List<String> messages = new ArrayList<>();
+ for (int i = t; i < total; i += 20) {
+ messages.add(prefix + i);
+ }
+ final RocketMQGrpcSink<String> sink =
+ RocketMQGrpcSink.<String>builder()
+ .setEndpoints(ENDPOINTS)
+ .setTopic(MAIN_TOPIC)
+ .setLiteTopic("flink-renew-lite-" + t)
+ .setValueOnlySerializer(new SimpleStringSchema())
+ .build();
+ sinkEnv.fromData(messages).sinkTo(sink);
+ }
+ sinkEnv.execute("lite-sink-renew");
+ System.out.println("[phase2] sink wrote " + total + " lite messages");
+
+ final StreamExecutionEnvironment env = localEnv();
+ final RocketMQGrpcSource<String> source =
+ RocketMQGrpcSource.<String>builder()
+ .setEndpoints(ENDPOINTS)
+ .setConsumerGroup(CONSUMER_GROUP)
+ .setMainTopic(MAIN_TOPIC)
+ .setConfig(
+ RocketMQGrpcSourceOptions.INVISIBLE_DURATION,
+ Duration.ofSeconds(20))
+ // Small batches so the downstream sleep keeps messages queued inside the
+ // split reader long enough to reach the renewal trigger point (20s - 18s).
+ .setConfig(RocketMQGrpcSourceOptions.MAX_MESSAGE_NUM, 4)
+ .setFetchConcurrency(4)
+ .setRenewalPolicyClass(CountingRenewalPolicy.class.getName())
+ .setRenewalAheadTime(Duration.ofSeconds(18))
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build();
+
+ final Configuration ackConfig = new Configuration();
+ ackConfig.set(RocketMQGrpcOptions.ENDPOINTS, ENDPOINTS);
+
+ final DataStream<String> stream =
+ env.fromSource(source, WatermarkStrategy.noWatermarks(), "grpc-source")
+ .process(
+ new RocketMQThrottleProcessFunction<String>(
+ ackConfig, value -> Optional.empty()))
+ .returns(String.class);
+ stream.map(
+ value -> {
+ Thread.sleep(300); // backpressure so messages linger in the split reader
+ return record(value);
+ });
+
+ ARRIVALS.clear();
+ RENEWALS.set(0);
+ runJobUntil(
+ env,
+ 240,
+ () -> {
+ for (int i = 0; i < total; i++) {
+ if (count(prefix + i) < 1) {
+ return false;
+ }
+ }
+ return true;
+ });
+
+ int duplicates = 0;
+ for (int i = 0; i < total; i++) {
+ if (count(prefix + i) > 1) {
+ duplicates++;
+ }
+ }
+ System.out.println(
+ "[phase2] renewals=" + RENEWALS.get() + ", duplicates=" + duplicates + "/" + total);
+ check(RENEWALS.get() > 0, "renewal policy never triggered");
+ check(duplicates == 0, "unexpected duplicates: " + duplicates);
+ System.out.println("[phase2] PASS: invisible-duration renewal fired, no redelivery");
+ }
+
+ // ------------------------------------------------------------------
+ // Helpers
+ // ------------------------------------------------------------------
+
+ /** Renewal policy that counts invocations and extends by another 20s. */
+ public static class CountingRenewalPolicy implements InvisibleDurationRenewalPolicy {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ @Nullable
+ public Duration renew(MessageView messageView, int renewalCount) {
+ RENEWALS.incrementAndGet();
+ return Duration.ofSeconds(20);
+ }
+ }
+
+ private static void runSinkJob(List<String> messages, String liteTopic) throws Exception {
+ final StreamExecutionEnvironment env = localEnv();
+ final RocketMQGrpcSink<String> sink =
+ RocketMQGrpcSink.<String>builder()
+ .setEndpoints(ENDPOINTS)
+ .setTopic(MAIN_TOPIC)
+ .setLiteTopic(liteTopic)
+ .setValueOnlySerializer(new SimpleStringSchema())
+ .build();
+ env.fromData(messages).sinkTo(sink);
+ env.execute("lite-sink-" + liteTopic);
+ }
+
+ private static StreamExecutionEnvironment localEnv() {
+ final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
+ env.setParallelism(1);
+ return env;
+ }
+
+ private static String record(String value) {
+ ARRIVALS.computeIfAbsent(value, k -> new CopyOnWriteArrayList<>())
+ .add(System.currentTimeMillis());
+ return value;
+ }
+
+ private static int count(String value) {
+ final List<Long> times = ARRIVALS.get(value);
+ return times == null ? 0 : times.size();
+ }
+
+ private static void runJobUntil(
+ StreamExecutionEnvironment env, int timeoutSeconds, Condition condition)
+ throws Exception {
+ final Thread jobThread =
+ new Thread(
+ () -> {
+ try {
+ env.execute("verify-job");
+ } catch (Exception e) {
+ System.out.println("job ended: " + e.getMessage());
+ }
+ });
+ jobThread.setDaemon(true);
+ jobThread.start();
+
+ final long deadline = System.currentTimeMillis() + timeoutSeconds * 1000L;
+ boolean satisfied = false;
+ int ticks = 0;
+ while (System.currentTimeMillis() < deadline) {
+ if (condition.test()) {
+ satisfied = true;
+ break;
+ }
+ if (++ticks % 15 == 0) {
+ System.out.println(
+ "[progress] distinct=" + ARRIVALS.size() + " after " + ticks + "s");
+ }
+ Thread.sleep(1000);
+ }
+ // Give in-flight acks a moment before tearing the job down.
+ Thread.sleep(3000);
+ env.close();
+ jobThread.interrupt();
+ jobThread.join(15000);
+ check(satisfied, "condition not satisfied within " + timeoutSeconds + "s");
+ }
+
+ private static void check(boolean condition, String message) {
+ if (!condition) {
+ throw new IllegalStateException("VERIFY FAILED: " + message);
+ }
+ }
+
+ @FunctionalInterface
+ private interface Condition {
+ boolean test();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageSerializerTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageSerializerTest.java
new file mode 100644
index 0000000..c8b2204
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/AckableMessageSerializerTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.base.StringSerializer;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataOutputSerializer;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link AckableMessageTypeInfo.Serializer}. */
+class AckableMessageSerializerTest {
+
+ private final AckableMessageTypeInfo.Serializer<String> serializer =
+ new AckableMessageTypeInfo.Serializer<>(StringSerializer.INSTANCE);
+
+ private static RocketMQReceiptHandle handle() {
+ return new RocketMQReceiptHandle(
+ "127.0.0.1:8080", "ns", "grp", "topic", "sub", "mid", "rh", 1);
+ }
+
+ @Test
+ void roundTripWithValue() throws IOException {
+ final AckableMessage<String> message = new AckableMessage<>("payload", handle());
+ assertThat(roundTrip(message)).isEqualTo(message);
+ }
+
+ @Test
+ void roundTripWithNullValue() throws IOException {
+ final AckableMessage<String> message = new AckableMessage<>(null, handle());
+ final AckableMessage<String> deserialized = roundTrip(message);
+ assertThat(deserialized.getValue()).isNull();
+ assertThat(deserialized).isEqualTo(message);
+ }
+
+ @Test
+ void snapshotIsCompatibleWithSameValueSerializer() {
+ final TypeSerializerSnapshot<AckableMessage<String>> snapshot =
+ serializer.snapshotConfiguration();
+ final AckableMessageTypeInfo.Serializer<String> other =
+ new AckableMessageTypeInfo.Serializer<>(StringSerializer.INSTANCE);
+ final TypeSerializerSchemaCompatibility<AckableMessage<String>> compatibility =
+ other.snapshotConfiguration().resolveSchemaCompatibility(snapshot);
+ assertThat(compatibility.isCompatibleAsIs()).isTrue();
+ }
+
+ private AckableMessage<String> roundTrip(AckableMessage<String> message) throws IOException {
+ final DataOutputSerializer out = new DataOutputSerializer(64);
+ serializer.serialize(message, out);
+ final DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer());
+ return serializer.deserialize(in);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClientTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClientTest.java
new file mode 100644
index 0000000..5c9dc5e
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQLiteAckClientTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for the reference counting behaviour of {@link RocketMQLiteAckClient#acquire}. */
+class RocketMQLiteAckClientTest {
+
+ private static Configuration config(String endpoints) {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcOptions.ENDPOINTS, endpoints);
+ return configuration;
+ }
+
+ @Test
+ void sameConfigurationSharesASingleReferenceCountedClient() {
+ final Configuration configuration = config("127.0.0.1:8080");
+ assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isZero();
+
+ final RocketMQLiteAckClient first = RocketMQLiteAckClient.acquire(configuration);
+ final RocketMQLiteAckClient second = RocketMQLiteAckClient.acquire(configuration);
+
+ // The same client is shared and the reference count is incremented per acquire.
+ assertThat(second).isSameAs(first);
+ assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isEqualTo(2);
+
+ RocketMQLiteAckClient.release(configuration);
+ assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isEqualTo(1);
+
+ RocketMQLiteAckClient.release(configuration);
+ assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isZero();
+ }
+
+ @Test
+ void differentConfigurationsGetDistinctClients() {
+ final Configuration a = config("host-a:8080");
+ final Configuration b = config("host-b:8080");
+
+ final RocketMQLiteAckClient clientA = RocketMQLiteAckClient.acquire(a);
+ final RocketMQLiteAckClient clientB = RocketMQLiteAckClient.acquire(b);
+ try {
+ assertThat(clientA).isNotSameAs(clientB);
+ assertThat(RocketMQLiteAckClient.getReferenceCount(a)).isEqualTo(1);
+ assertThat(RocketMQLiteAckClient.getReferenceCount(b)).isEqualTo(1);
+ } finally {
+ RocketMQLiteAckClient.release(a);
+ RocketMQLiteAckClient.release(b);
+ }
+
+ assertThat(RocketMQLiteAckClient.getReferenceCount(a)).isZero();
+ assertThat(RocketMQLiteAckClient.getReferenceCount(b)).isZero();
+ }
+
+ @Test
+ void releaseWithoutAcquireIsANoOp() {
+ final Configuration configuration = config("no-acquire:8080");
+ RocketMQLiteAckClient.release(configuration);
+ assertThat(RocketMQLiteAckClient.getReferenceCount(configuration)).isZero();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodecTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodecTest.java
new file mode 100644
index 0000000..7854705
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleCodecTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.rocketmq.client.apis.message.MessageId;
+import org.apache.rocketmq.client.apis.message.MessageView;
+import org.apache.rocketmq.client.java.message.MessageIdCodec;
+import org.apache.rocketmq.client.java.message.MessageViewImpl;
+import org.apache.rocketmq.client.java.route.Endpoints;
+import org.junit.jupiter.api.Test;
+
+import java.lang.reflect.Proxy;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link RocketMQReceiptHandleCodec}. */
+class RocketMQReceiptHandleCodecTest {
+
+ private static RocketMQReceiptHandle newHandle(String liteTopic) {
+ final String messageId = MessageIdCodec.getInstance().nextMessageId().toString();
+ return new RocketMQReceiptHandle(
+ "127.0.0.1:8080",
+ "ns-1",
+ "GID-test",
+ "main-topic",
+ liteTopic,
+ messageId,
+ "receipt-handle-abc",
+ 2);
+ }
+
+ @Test
+ void toAckableThenExtractRoundTripsAllFields() {
+ final RocketMQReceiptHandle original = newHandle("sub-topic-a");
+
+ final MessageView view = RocketMQReceiptHandleCodec.toAckable(original);
+ final RocketMQReceiptHandle roundTripped =
+ RocketMQReceiptHandleCodec.extract(
+ view, original.getNamespace(), original.getConsumerGroup());
+
+ assertThat(roundTripped).isEqualTo(original);
+ }
+
+ @Test
+ void toAckableSupportsNullLiteTopic() {
+ final RocketMQReceiptHandle original = newHandle(null);
+
+ final MessageView view = RocketMQReceiptHandleCodec.toAckable(original);
+ final RocketMQReceiptHandle roundTripped =
+ RocketMQReceiptHandleCodec.extract(
+ view, original.getNamespace(), original.getConsumerGroup());
+
+ assertThat(roundTripped.getLiteTopic()).isNull();
+ assertThat(roundTripped).isEqualTo(original);
+ }
+
+ @Test
+ void reconstructedViewCarriesParseableMessageIdAndEndpoints() {
+ final RocketMQReceiptHandle original = newHandle("sub-topic-a");
+
+ final MessageViewImpl view =
+ (MessageViewImpl) RocketMQReceiptHandleCodec.toAckable(original);
+
+ // MessageId round-trips through the SDK codec.
+ final MessageId expectedId = MessageIdCodec.getInstance().decode(original.getMessageId());
+ assertThat(view.getMessageId()).isEqualTo(expectedId);
+
+ // Endpoints round-trip through the string form used by the handle.
+ final Endpoints endpoints = view.getEndpoints();
+ assertThat(endpoints).isNotNull();
+ assertThat(endpoints).isEqualTo(new Endpoints(original.getEndpoint()));
+
+ assertThat(view.getTopic()).isEqualTo(original.getTopic());
+ assertThat(view.getReceiptHandle()).isEqualTo(original.getReceiptHandle());
+ assertThat(view.getDeliveryAttempt()).isEqualTo(original.getDeliveryAttempt());
+ }
+
+ @Test
+ void extractRejectsUnexpectedMessageViewType() {
+ final MessageView notImpl =
+ (MessageView)
+ Proxy.newProxyInstance(
+ MessageView.class.getClassLoader(),
+ new Class[] {MessageView.class},
+ (proxy, method, args) -> {
+ throw new UnsupportedOperationException();
+ });
+
+ assertThatThrownBy(() -> RocketMQReceiptHandleCodec.extract(notImpl, "ns", "grp"))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("internal message type");
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleSerializerTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleSerializerTest.java
new file mode 100644
index 0000000..4d7e56b
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQReceiptHandleSerializerTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataOutputSerializer;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link RocketMQReceiptHandle.Serializer}. */
+class RocketMQReceiptHandleSerializerTest {
+
+ private final RocketMQReceiptHandle.Serializer serializer =
+ RocketMQReceiptHandle.Serializer.INSTANCE;
+
+ @Test
+ void roundTripWithLiteTopic() throws IOException {
+ final RocketMQReceiptHandle handle =
+ new RocketMQReceiptHandle(
+ "127.0.0.1:8080", "ns", "grp", "topic", "sub", "mid", "rh", 3);
+
+ assertThat(roundTrip(handle)).isEqualTo(handle);
+ }
+
+ @Test
+ void roundTripWithNullLiteTopic() throws IOException {
+ final RocketMQReceiptHandle handle =
+ new RocketMQReceiptHandle(
+ "127.0.0.1:8080", "", "grp", "topic", null, "mid", "rh", 0);
+
+ final RocketMQReceiptHandle deserialized = roundTrip(handle);
+ assertThat(deserialized.getLiteTopic()).isNull();
+ assertThat(deserialized).isEqualTo(handle);
+ }
+
+ @Test
+ void snapshotIsCompatibleAsIs() {
+ final TypeSerializerSnapshot<RocketMQReceiptHandle> snapshot =
+ serializer.snapshotConfiguration();
+ final TypeSerializerSchemaCompatibility<RocketMQReceiptHandle> compatibility =
+ serializer.snapshotConfiguration().resolveSchemaCompatibility(snapshot);
+ assertThat(compatibility.isCompatibleAsIs()).isTrue();
+ }
+
+ private RocketMQReceiptHandle roundTrip(RocketMQReceiptHandle handle) throws IOException {
+ final DataOutputSerializer out = new DataOutputSerializer(64);
+ serializer.serialize(handle, out);
+ final DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer());
+ return serializer.deserialize(in);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunctionTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunctionTest.java
new file mode 100644
index 0000000..5469316
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/ack/RocketMQThrottleProcessFunctionTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.ack;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.Collector;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for the throttle branching and safeguards of {@link RocketMQThrottleProcessFunction}. */
+class RocketMQThrottleProcessFunctionTest {
+
+ private static RocketMQReceiptHandle handle(String messageId, int deliveryAttempt) {
+ return new RocketMQReceiptHandle(
+ "127.0.0.1:8080",
+ "ns",
+ "grp",
+ "topic",
+ "sub",
+ messageId,
+ "rh-" + messageId,
+ deliveryAttempt);
+ }
+
+ @Test
+ void emptyPolicyResultAcknowledges() throws Exception {
+ final RecordingThrottleFunction fn =
+ new RecordingThrottleFunction(
+ value -> Optional.empty(), 16, Duration.ofMinutes(30));
+ final ListCollector out = new ListCollector();
+
+ fn.processElement(new AckableMessage<>("v", handle("m1", 0)), null, out);
+
+ assertThat(fn.acked).containsExactly("m1");
+ assertThat(fn.changed).isEmpty();
+ assertThat(out.values).containsExactly("v");
+ }
+
+ @Test
+ void policyDelayChangesInvisibleDuration() throws Exception {
+ final Duration delay = Duration.ofSeconds(10);
+ final RecordingThrottleFunction fn =
+ new RecordingThrottleFunction(
+ value -> Optional.of(delay), 16, Duration.ofMinutes(30));
+ final ListCollector out = new ListCollector();
+
+ fn.processElement(new AckableMessage<>("v", handle("m1", 0)), null, out);
+
+ assertThat(fn.changed).containsEntry("m1", delay);
+ assertThat(fn.acked).isEmpty();
+ // The value is always forwarded downstream.
+ assertThat(out.values).containsExactly("v");
+ }
+
+ @Test
+ void requestedDelayIsCappedToMaxInvisibleDuration() throws Exception {
+ final Duration cap = Duration.ofSeconds(30);
+ final RecordingThrottleFunction fn =
+ new RecordingThrottleFunction(value -> Optional.of(Duration.ofHours(1)), 16, cap);
+ final ListCollector out = new ListCollector();
+
+ fn.processElement(new AckableMessage<>("v", handle("m1", 0)), null, out);
+
+ assertThat(fn.changed).containsEntry("m1", cap);
+ }
+
+ @Test
+ void deferSafeguardForcesAckOnceMaxDeliveryAttemptReached() throws Exception {
+ final int maxAttempt = 3;
+ final RecordingThrottleFunction fn =
+ new RecordingThrottleFunction(
+ value -> Optional.of(Duration.ofSeconds(10)),
+ maxAttempt,
+ Duration.ofMinutes(30));
+ final ListCollector out = new ListCollector();
+
+ // Below the bound: still deferred.
+ fn.processElement(new AckableMessage<>("v", handle("m1", maxAttempt - 1)), null, out);
+ assertThat(fn.changed).containsKey("m1");
+ assertThat(fn.acked).isEmpty();
+
+ // At the bound: the safeguard trips and the message is acknowledged instead.
+ fn.processElement(new AckableMessage<>("v", handle("m2", maxAttempt)), null, out);
+ assertThat(fn.acked).containsExactly("m2");
+ }
+
+ private static final class RecordingThrottleFunction
+ extends RocketMQThrottleProcessFunction<String> {
+
+ private static final long serialVersionUID = 1L;
+
+ private final List<String> acked = new ArrayList<>();
+ private final java.util.Map<String, Duration> changed = new java.util.LinkedHashMap<>();
+
+ RecordingThrottleFunction(
+ MessageThrottlePolicy<String> policy,
+ int maxDeliveryAttempt,
+ Duration maxInvisible) {
+ super(new Configuration(), policy, maxDeliveryAttempt, maxInvisible);
+ }
+
+ @Override
+ protected void ack(RocketMQReceiptHandle handle) {
+ acked.add(handle.getMessageId());
+ }
+
+ @Override
+ protected void changeInvisibleDuration(
+ RocketMQReceiptHandle handle, Duration invisibleDuration) {
+ changed.put(handle.getMessageId(), invisibleDuration);
+ }
+ }
+
+ private static final class ListCollector implements Collector<String> {
+
+ private final List<String> values = new ArrayList<>();
+
+ @Override
+ public void collect(String record) {
+ values.add(record);
+ }
+
+ @Override
+ public void close() {}
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolversTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolversTest.java
new file mode 100644
index 0000000..375854f
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/common/CredentialsResolversTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.common;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.RocketMQGrpcOptions;
+import org.apache.flink.util.FlinkRuntimeException;
+
+import org.apache.rocketmq.client.apis.ClientConfiguration;
+import org.apache.rocketmq.client.apis.ClientConfigurationBuilder;
+import org.apache.rocketmq.client.apis.SessionCredentialsProvider;
+import org.apache.rocketmq.client.apis.StaticSessionCredentialsProvider;
+import org.junit.jupiter.api.Test;
+
+import javax.annotation.Nullable;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link CredentialsResolvers}. */
+class CredentialsResolversTest {
+
+ private static final SessionCredentialsProvider RESOLVED_PROVIDER =
+ new StaticSessionCredentialsProvider("resolved-ak", "resolved-sk");
+
+ @Test
+ void createFromConfigurationReturnsNullWithoutResolverClass() {
+ assertThat(CredentialsResolvers.createFromConfiguration(new Configuration())).isNull();
+ }
+
+ @Test
+ void createFromConfigurationInstantiatesAndConfiguresTheResolver() {
+ final Configuration configuration = new Configuration();
+ configuration.set(
+ RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS, RecordingResolver.class.getName());
+
+ final CredentialsResolver resolver =
+ CredentialsResolvers.createFromConfiguration(configuration);
+
+ assertThat(resolver).isInstanceOf(RecordingResolver.class);
+ assertThat(((RecordingResolver) resolver).configured).isSameAs(configuration);
+ }
+
+ @Test
+ void createFromConfigurationFailsForNonResolverClass() {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS, String.class.getName());
+
+ assertThatThrownBy(() -> CredentialsResolvers.createFromConfiguration(configuration))
+ .isInstanceOf(FlinkRuntimeException.class)
+ .hasMessageContaining(String.class.getName());
+ }
+
+ @Test
+ void createFromConfigurationFailsForMissingClass() {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcOptions.CREDENTIALS_RESOLVER_CLASS, "does.not.Exist");
+
+ assertThatThrownBy(() -> CredentialsResolvers.createFromConfiguration(configuration))
+ .isInstanceOf(FlinkRuntimeException.class)
+ .hasMessageContaining("does.not.Exist");
+ }
+
+ @Test
+ void resolverTakesPrecedenceOverStaticKeys() {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcOptions.ACCESS_KEY, "static-ak");
+ configuration.set(RocketMQGrpcOptions.SECRET_KEY, "static-sk");
+ final RecordingResolver resolver = new RecordingResolver();
+
+ final ClientConfigurationBuilder builder = newBuilder();
+ CredentialsResolvers.applyCredentials(builder, "host-a:8080", configuration, resolver);
+
+ assertThat(resolver.resolvedEndpoint).isEqualTo("host-a:8080");
+ assertThat(builder.build().getCredentialsProvider()).contains(RESOLVED_PROVIDER);
+ }
+
+ @Test
+ void staticKeysAreUsedWithoutAResolver() {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcOptions.ACCESS_KEY, "static-ak");
+ configuration.set(RocketMQGrpcOptions.SECRET_KEY, "static-sk");
+
+ final ClientConfigurationBuilder builder = newBuilder();
+ CredentialsResolvers.applyCredentials(builder, "host-a:8080", configuration, null);
+
+ assertThat(builder.build().getCredentialsProvider())
+ .containsInstanceOf(StaticSessionCredentialsProvider.class);
+ }
+
+ @Test
+ void noCredentialsAreSetWhenTheResolverReturnsNull() {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcOptions.ACCESS_KEY, "static-ak");
+ configuration.set(RocketMQGrpcOptions.SECRET_KEY, "static-sk");
+
+ final ClientConfigurationBuilder builder = newBuilder();
+ CredentialsResolvers.applyCredentials(
+ builder, "host-a:8080", configuration, endpoint -> null);
+
+ // The resolver explicitly resolved "no credentials"; the static keys must not be used.
+ assertThat(builder.build().getCredentialsProvider()).isEmpty();
+ }
+
+ @Test
+ void noCredentialsAreSetWithoutResolverAndWithIncompleteStaticKeys() {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcOptions.ACCESS_KEY, "static-ak");
+
+ final ClientConfigurationBuilder builder = newBuilder();
+ CredentialsResolvers.applyCredentials(builder, "host-a:8080", configuration, null);
+
+ assertThat(builder.build().getCredentialsProvider()).isEmpty();
+ }
+
+ private static ClientConfigurationBuilder newBuilder() {
+ return ClientConfiguration.newBuilder().setEndpoints("host-a:8080");
+ }
+
+ /** A resolver that records its interactions for assertions. */
+ public static class RecordingResolver implements CredentialsResolver {
+
+ @Nullable private Configuration configured;
+ @Nullable private String resolvedEndpoint;
+
+ @Override
+ public void configure(Configuration configuration) {
+ this.configured = configuration;
+ }
+
+ @Override
+ public SessionCredentialsProvider resolve(String endpoint) {
+ this.resolvedEndpoint = endpoint;
+ return RESOLVED_PROVIDER;
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilderTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilderTest.java
new file mode 100644
index 0000000..2d559b5
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/sink/RocketMQGrpcSinkBuilderTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.sink;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link RocketMQGrpcSinkBuilder}. */
+class RocketMQGrpcSinkBuilderTest {
+
+ @Test
+ void buildSucceedsWithValueOnlySerializerTest() {
+ final RocketMQGrpcSink<String> sink =
+ RocketMQGrpcSink.<String>builder()
+ .setEndpoints("127.0.0.1:8080")
+ .setTopic("topic")
+ .setLiteTopic("lite-topic")
+ .setValueOnlySerializer(new SimpleStringSchema())
+ .build();
+ assertThat(sink).isNotNull();
+ }
+
+ @Test
+ void buildFailsWithoutEndpointsTest() {
+ assertThatThrownBy(
+ () ->
+ RocketMQGrpcSink.<String>builder()
+ .setTopic("topic")
+ .setLiteTopic("lite-topic")
+ .setValueOnlySerializer(new SimpleStringSchema())
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("endpoints");
+ }
+
+ @Test
+ void buildFailsWithoutSerializerTest() {
+ assertThatThrownBy(
+ () ->
+ RocketMQGrpcSink.<String>builder()
+ .setEndpoints("127.0.0.1:8080")
+ .setTopic("topic")
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("serializer");
+ }
+
+ @Test
+ void valueOnlySerializerRequiresTopicTest() {
+ assertThatThrownBy(
+ () ->
+ RocketMQGrpcSink.<String>builder()
+ .setEndpoints("127.0.0.1:8080")
+ .setValueOnlySerializer(new SimpleStringSchema()))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("topic");
+ }
+
+ @Test
+ void valueOnlySerializerRequiresLiteTopicTest() {
+ assertThatThrownBy(
+ () ->
+ RocketMQGrpcSink.<String>builder()
+ .setEndpoints("127.0.0.1:8080")
+ .setTopic("topic")
+ .setValueOnlySerializer(new SimpleStringSchema()))
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("lite topic");
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPoliciesTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPoliciesTest.java
new file mode 100644
index 0000000..93225f8
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/InvisibleDurationRenewalPoliciesTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView;
+import org.apache.flink.util.FlinkRuntimeException;
+
+import org.junit.jupiter.api.Test;
+
+import javax.annotation.Nullable;
+
+import java.time.Duration;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Unit tests for {@link InvisibleDurationRenewalPolicies}. */
+class InvisibleDurationRenewalPoliciesTest {
+
+ @Test
+ void returnsNullWhenNoPolicyConfiguredTest() {
+ assertThat(InvisibleDurationRenewalPolicies.createFromConfiguration(new Configuration()))
+ .isNull();
+ }
+
+ @Test
+ void instantiatesAndConfiguresPolicyTest() {
+ final Configuration configuration = new Configuration();
+ configuration.set(
+ RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS,
+ RecordingRenewalPolicy.class.getName());
+ final InvisibleDurationRenewalPolicy policy =
+ InvisibleDurationRenewalPolicies.createFromConfiguration(configuration);
+ assertThat(policy).isInstanceOf(RecordingRenewalPolicy.class);
+ assertThat(((RecordingRenewalPolicy) policy).configured).isTrue();
+ }
+
+ @Test
+ void failsForUnknownClassTest() {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS, "does.not.Exist");
+ assertThatThrownBy(
+ () ->
+ InvisibleDurationRenewalPolicies.createFromConfiguration(
+ configuration))
+ .isInstanceOf(FlinkRuntimeException.class);
+ }
+
+ @Test
+ void failsForNonPolicyClassTest() {
+ final Configuration configuration = new Configuration();
+ configuration.set(RocketMQGrpcSourceOptions.RENEWAL_POLICY_CLASS, String.class.getName());
+ assertThatThrownBy(
+ () ->
+ InvisibleDurationRenewalPolicies.createFromConfiguration(
+ configuration))
+ .isInstanceOf(FlinkRuntimeException.class);
+ }
+
+ /** A renewal policy that records whether it was configured. */
+ public static class RecordingRenewalPolicy implements InvisibleDurationRenewalPolicy {
+
+ private static final long serialVersionUID = 1L;
+
+ boolean configured;
+
+ @Override
+ public void configure(Configuration configuration) {
+ this.configured = true;
+ }
+
+ @Nullable
+ @Override
+ public Duration renew(MessageView messageView, int renewalCount) {
+ return null;
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilderTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilderTest.java
new file mode 100644
index 0000000..21a8c6a
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceBuilderTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.api.common.serialization.SimpleStringSchema;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link RocketMQGrpcSourceBuilder}. */
+class RocketMQGrpcSourceBuilderTest {
+
+ @Test
+ void buildSucceedsWithAllRequiredOptionsTest() {
+ final RocketMQGrpcSource<String> source =
+ RocketMQGrpcSource.<String>builder()
+ .setEndpoints("127.0.0.1:8080")
+ .setConsumerGroup("group")
+ .setMainTopic("topic")
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build();
+ assertThat(source).isNotNull();
+ }
+
+ @Test
+ void buildFailsWithoutEndpointsTest() {
+ assertThatThrownBy(
+ () ->
+ RocketMQGrpcSource.<String>builder()
+ .setConsumerGroup("group")
+ .setMainTopic("topic")
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("endpoints");
+ }
+
+ @Test
+ void buildFailsWithoutConsumerGroupTest() {
+ assertThatThrownBy(
+ () ->
+ RocketMQGrpcSource.<String>builder()
+ .setEndpoints("127.0.0.1:8080")
+ .setMainTopic("topic")
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("consumer group");
+ }
+
+ @Test
+ void buildFailsWithoutMainTopicTest() {
+ assertThatThrownBy(
+ () ->
+ RocketMQGrpcSource.<String>builder()
+ .setEndpoints("127.0.0.1:8080")
+ .setConsumerGroup("group")
+ .setValueOnlyDeserializer(new SimpleStringSchema())
+ .build())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("main topic");
+ }
+
+ @Test
+ void buildFailsWithoutDeserializerTest() {
+ assertThatThrownBy(
+ () ->
+ RocketMQGrpcSource.<String>builder()
+ .setEndpoints("127.0.0.1:8080")
+ .setConsumerGroup("group")
+ .setMainTopic("topic")
+ .build())
+ .isInstanceOf(NullPointerException.class)
+ .hasMessageContaining("deserializer");
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceEnumStateSerializerTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceEnumStateSerializerTest.java
new file mode 100644
index 0000000..39917d9
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceEnumStateSerializerTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumState;
+import org.apache.flink.connector.rocketmq.grpc.source.enumerator.RocketMQGrpcSourceEnumStateSerializer;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link RocketMQGrpcSourceEnumStateSerializer}. */
+class RocketMQGrpcSourceEnumStateSerializerTest {
+
+ @Test
+ void serializeAndDeserializeRoundTripTest() throws Exception {
+ final RocketMQGrpcSourceEnumStateSerializer serializer =
+ new RocketMQGrpcSourceEnumStateSerializer();
+
+ final byte[] serialized = serializer.serialize(new RocketMQGrpcSourceEnumState());
+ final RocketMQGrpcSourceEnumState deserialized =
+ serializer.deserialize(serializer.getVersion(), serialized);
+
+ assertThat(deserialized).isNotNull();
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceSplitSerializerTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceSplitSerializerTest.java
new file mode 100644
index 0000000..3a3d1f5
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/RocketMQGrpcSourceSplitSerializerTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source;
+
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplit;
+import org.apache.flink.connector.rocketmq.grpc.source.split.RocketMQGrpcSourceSplitSerializer;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link RocketMQGrpcSourceSplitSerializer}. */
+class RocketMQGrpcSourceSplitSerializerTest {
+
+ @Test
+ void serializeAndDeserializeRoundTripTest() throws Exception {
+ final RocketMQGrpcSourceSplitSerializer serializer =
+ new RocketMQGrpcSourceSplitSerializer();
+ final RocketMQGrpcSourceSplit split = new RocketMQGrpcSourceSplit();
+
+ final byte[] serialized = serializer.serialize(split);
+ final RocketMQGrpcSourceSplit deserialized =
+ serializer.deserialize(serializer.getVersion(), serialized);
+
+ assertThat(deserialized).isEqualTo(split);
+ assertThat(deserialized.splitId()).isEqualTo(RocketMQGrpcSourceSplit.SPLIT_ID);
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImplTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImplTest.java
new file mode 100644
index 0000000..29bee9a
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/source/reader/MessageViewImplTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.source.reader;
+
+import org.apache.rocketmq.client.apis.message.MessageId;
+import org.junit.jupiter.api.Test;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.FutureTask;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Unit tests for the renewal/emit bookkeeping of {@link MessageViewImpl}. */
+class MessageViewImplTest {
+
+ @Test
+ void markEmittedCancelsPendingRenewalTest() {
+ final MessageViewImpl view = new MessageViewImpl(new TestingSdkMessageView());
+ final FutureTask<Void> future = new FutureTask<>(() -> null);
+ view.setRenewalFuture(future);
+
+ assertThat(view.isEmitted()).isFalse();
+ view.markEmitted();
+
+ assertThat(view.isEmitted()).isTrue();
+ assertThat(future.isCancelled()).isTrue();
+ }
+
+ @Test
+ void renewalFutureSetAfterEmitIsCancelledTest() {
+ final MessageViewImpl view = new MessageViewImpl(new TestingSdkMessageView());
+ view.markEmitted();
+
+ final FutureTask<Void> future = new FutureTask<>(() -> null);
+ view.setRenewalFuture(future);
+
+ assertThat(future.isCancelled()).isTrue();
+ }
+
+ @Test
+ void copiesBodyAndExposesAttributesTest() {
+ final MessageViewImpl view = new MessageViewImpl(new TestingSdkMessageView());
+
+ assertThat(view.getBody()).isEqualTo("body".getBytes(StandardCharsets.UTF_8));
+ assertThat(view.getTopic()).isEqualTo("topic");
+ assertThat(view.getTag()).isNull();
+ assertThat(view.getDeliveryAttempt()).isEqualTo(2);
+ assertThat(view.getEventTime()).isEqualTo(1234L);
+ }
+
+ /** A minimal SDK message view stub. */
+ private static class TestingSdkMessageView
+ implements org.apache.rocketmq.client.apis.message.MessageView {
+
+ @Override
+ public MessageId getMessageId() {
+ return null;
+ }
+
+ @Override
+ public String getTopic() {
+ return "topic";
+ }
+
+ @Override
+ public ByteBuffer getBody() {
+ return ByteBuffer.wrap("body".getBytes(StandardCharsets.UTF_8));
+ }
+
+ @Override
+ public Map<String, String> getProperties() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Optional<String> getTag() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Collection<String> getKeys() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public Optional<String> getMessageGroup() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<String> getLiteTopic() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<Long> getDeliveryTimestamp() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<Integer> getPriority() {
+ return Optional.empty();
+ }
+
+ @Override
+ public String getBornHost() {
+ return "localhost";
+ }
+
+ @Override
+ public long getBornTimestamp() {
+ return 1234L;
+ }
+
+ @Override
+ public int getDeliveryAttempt() {
+ return 2;
+ }
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactoryTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactoryTest.java
new file mode 100644
index 0000000..67aa3d4
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcDynamicTableFactoryTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.table;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.Column;
+import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.utils.FactoryMocks;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link RocketMQGrpcDynamicTableFactory}. */
+class RocketMQGrpcDynamicTableFactoryTest {
+
+ private static final ResolvedSchema SCHEMA =
+ ResolvedSchema.of(Column.physical("f0", DataTypes.STRING()));
+
+ private static Map<String, String> baseOptions() {
+ final Map<String, String> options = new HashMap<>();
+ options.put("connector", RocketMQGrpcDynamicTableFactory.IDENTIFIER);
+ options.put("endpoints", "127.0.0.1:8080");
+ options.put("topic", "test-topic");
+ options.put("sink.lite-topic", "test-lite-topic");
+ options.put("format", "test-format");
+ options.put("test-format.delimiter", ",");
+ return options;
+ }
+
+ @Test
+ void createTableSourceTest() {
+ final Map<String, String> options = baseOptions();
+ options.put("source.consumer-group", "test-group");
+
+ final DynamicTableSource source = FactoryMocks.createTableSource(SCHEMA, options);
+ assertThat(source).isInstanceOf(RocketMQGrpcDynamicTableSource.class);
+ }
+
+ @Test
+ void createTableSinkTest() {
+ final DynamicTableSink sink = FactoryMocks.createTableSink(SCHEMA, baseOptions());
+ assertThat(sink).isInstanceOf(RocketMQGrpcDynamicTableSink.class);
+ }
+
+ @Test
+ void createTableSourceFailsWithoutConsumerGroupTest() {
+ assertThatThrownBy(() -> FactoryMocks.createTableSource(SCHEMA, baseOptions()))
+ .isInstanceOf(ValidationException.class)
+ .hasStackTraceContaining("consumer-group");
+ }
+
+ @Test
+ void createTableSinkFailsWithoutLiteTopicTest() {
+ final Map<String, String> options = baseOptions();
+ options.remove("sink.lite-topic");
+ assertThatThrownBy(() -> FactoryMocks.createTableSink(SCHEMA, options))
+ .hasStackTraceContaining("lite-topic");
+ }
+}
diff --git a/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverterTest.java b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverterTest.java
new file mode 100644
index 0000000..135adb8
--- /dev/null
+++ b/flink-connector-rocketmq-grpc/src/test/java/org/apache/flink/connector/rocketmq/grpc/table/RocketMQGrpcRowDataConverterTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.connector.rocketmq.grpc.table;
+
+import org.apache.flink.api.common.serialization.DeserializationSchema;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.connector.rocketmq.grpc.source.reader.MessageView;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.util.Collector;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Unit tests for the metadata handling of {@link RocketMQGrpcRowDataConverter}. */
+class RocketMQGrpcRowDataConverterTest {
+
+ @Test
+ void appendsRequestedMetadataColumnsTest() throws Exception {
+ final List<RocketMQGrpcReadableMetadata.MetadataConverter> converters =
+ Arrays.asList(
+ RocketMQGrpcReadableMetadata.TAG.getConverter(),
+ RocketMQGrpcReadableMetadata.DELIVERY_ATTEMPT.getConverter(),
+ RocketMQGrpcReadableMetadata.BORN_TIMESTAMP.getConverter());
+ final RocketMQGrpcRowDataConverter converter =
+ RocketMQGrpcRowDataConverter.forSource(
+ new BodyAsStringDeserializationSchema(),
+ converters,
+ TypeInformation.of(RowData.class));
+ converter.open((DeserializationSchema.InitializationContext) null);
+
+ final List<RowData> output = new ArrayList<>();
+ converter.deserialize(new TestingMessageView(), new ListCollector(output));
+
+ assertThat(output).hasSize(1);
+ final GenericRowData row = (GenericRowData) output.get(0);
+ assertThat(row.getArity()).isEqualTo(4);
+ assertThat(row.getField(0)).isEqualTo(StringData.fromString("body"));
+ assertThat(row.getField(1)).isEqualTo(StringData.fromString("tagA"));
+ assertThat(row.getField(2)).isEqualTo(7);
+ assertThat(row.getField(3)).isEqualTo(TimestampData.fromEpochMillis(1234L));
+ }
+
+ @Test
+ void forwardsPhysicalRowWithoutMetadataTest() throws Exception {
+ final RocketMQGrpcRowDataConverter converter =
+ RocketMQGrpcRowDataConverter.forSource(
+ new BodyAsStringDeserializationSchema(),
+ Collections.emptyList(),
+ TypeInformation.of(RowData.class));
+ converter.open((DeserializationSchema.InitializationContext) null);
+
+ final List<RowData> output = new ArrayList<>();
+ converter.deserialize(new TestingMessageView(), new ListCollector(output));
+
+ assertThat(output).hasSize(1);
+ final GenericRowData row = (GenericRowData) output.get(0);
+ assertThat(row.getArity()).isEqualTo(1);
+ assertThat(row.getField(0)).isEqualTo(StringData.fromString("body"));
+ }
+
+ /** Decodes the message body into a single-field row containing the body as a string. */
+ private static class BodyAsStringDeserializationSchema
+ implements DeserializationSchema<RowData> {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public RowData deserialize(byte[] message) {
+ return GenericRowData.of(
+ StringData.fromString(new String(message, StandardCharsets.UTF_8)));
+ }
+
+ @Override
+ public boolean isEndOfStream(RowData nextElement) {
+ return false;
+ }
+
+ @Override
+ public TypeInformation<RowData> getProducedType() {
+ return TypeInformation.of(RowData.class);
+ }
+ }
+
+ private static class ListCollector implements Collector<RowData> {
+
+ private final List<RowData> output;
+
+ private ListCollector(List<RowData> output) {
+ this.output = output;
+ }
+
+ @Override
+ public void collect(RowData record) {
+ output.add(record);
+ }
+
+ @Override
+ public void close() {}
+ }
+
+ /** A connector-level message view with fixed attributes. */
+ private static class TestingMessageView implements MessageView {
+
+ @Override
+ public String getMessageId() {
+ return "MSG-1";
+ }
+
+ @Override
+ public String getTopic() {
+ return "topic";
+ }
+
+ @Override
+ public String getTag() {
+ return "tagA";
+ }
+
+ @Override
+ public Collection<String> getKeys() {
+ return Arrays.asList("k1", "k2");
+ }
+
+ @Override
+ public byte[] getBody() {
+ return "body".getBytes(StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public int getDeliveryAttempt() {
+ return 7;
+ }
+
+ @Override
+ public long getEventTime() {
+ return 1234L;
+ }
+
+ @Override
+ public Map<String, String> getProperties() {
+ return Collections.singletonMap("p1", "v1");
+ }
+ }
+}
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/common/config/RocketMQConfigValidator.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/common/config/RocketMQConfigValidator.java
index 39ada10..197d6c2 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/common/config/RocketMQConfigValidator.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/common/config/RocketMQConfigValidator.java
@@ -97,8 +97,10 @@ public class RocketMQConfigValidator {
}
public RocketMQConfigValidator build() {
- List<Set<ConfigOption<?>>> conflict = Collections.unmodifiableList(new ArrayList<>(conflictOptions));
- Set<ConfigOption<?>> required = Collections.unmodifiableSet(new HashSet<>(requiredOptions));
+ List<Set<ConfigOption<?>>> conflict =
+ Collections.unmodifiableList(new ArrayList<>(conflictOptions));
+ Set<ConfigOption<?>> required =
+ Collections.unmodifiableSet(new HashSet<>(requiredOptions));
return new RocketMQConfigValidator(conflict, required);
}
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSink.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSink.java
index f2562e5..3c89a27 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSink.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSink.java
@@ -21,13 +21,13 @@ import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.connector.sink2.Committer;
import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector;
import org.apache.flink.connector.rocketmq.sink.committer.RocketMQCommitter;
import org.apache.flink.connector.rocketmq.sink.committer.SendCommittable;
import org.apache.flink.connector.rocketmq.sink.committer.SendCommittableSerializer;
import org.apache.flink.connector.rocketmq.sink.writer.RocketMQWriter;
import org.apache.flink.connector.rocketmq.sink.writer.serializer.RocketMQSerializationSchema;
import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector;
@PublicEvolving
public class RocketMQSink<IN> implements TwoPhaseCommittingSink<IN, SendCommittable> {
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkBuilder.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkBuilder.java
index 5895612..21e194c 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkBuilder.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkBuilder.java
@@ -26,10 +26,10 @@ import org.apache.flink.connector.base.DeliveryGuarantee;
import org.apache.flink.connector.rocketmq.common.config.RocketMQConfigBuilder;
import org.apache.flink.connector.rocketmq.common.config.RocketMQConfigValidator;
import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions;
-import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector;
import org.apache.flink.connector.rocketmq.sink.writer.serializer.RocketMQSerializationSchema;
import org.apache.flink.connector.rocketmq.source.RocketMQSource;
import org.apache.flink.connector.rocketmq.source.RocketMQSourceOptions;
+import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector;
import org.apache.commons.lang3.StringUtils;
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkOptions.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkOptions.java
index 61e7263..a220e50 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkOptions.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/RocketMQSinkOptions.java
@@ -97,7 +97,9 @@ public class RocketMQSinkOptions extends RocketMQOptions {
ConfigOptions.key("rocketmq.sink.tag.dynamic.column").stringType().noDefaultValue();
public static final ConfigOption<Boolean> OPTIONAL_WRITE_DYNAMIC_TAG_COLUMN_WRITE_INCLUDED =
- ConfigOptions.key("rocketmq.sink.tag.dynamic.write.included").booleanType().defaultValue(true);
+ ConfigOptions.key("rocketmq.sink.tag.dynamic.write.included")
+ .booleanType()
+ .defaultValue(true);
public static final ConfigOption<String> OPTIONAL_WRITE_KEY_COLUMNS =
ConfigOptions.key("rocketmq.sink.key.columns").stringType().noDefaultValue();
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/writer/RocketMQWriter.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/writer/RocketMQWriter.java
index a5f2a6f..ae1b4e6 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/writer/RocketMQWriter.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/sink/writer/RocketMQWriter.java
@@ -22,7 +22,6 @@ import org.apache.flink.api.connector.sink2.Sink;
import org.apache.flink.api.connector.sink2.TwoPhaseCommittingSink;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.base.DeliveryGuarantee;
-import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector;
import org.apache.flink.connector.rocketmq.sink.InnerProducer;
import org.apache.flink.connector.rocketmq.sink.InnerProducerImpl;
import org.apache.flink.connector.rocketmq.sink.RocketMQSinkOptions;
@@ -30,6 +29,7 @@ import org.apache.flink.connector.rocketmq.sink.committer.SendCommittable;
import org.apache.flink.connector.rocketmq.sink.writer.context.RocketMQSinkContext;
import org.apache.flink.connector.rocketmq.sink.writer.context.RocketMQSinkContextImpl;
import org.apache.flink.connector.rocketmq.sink.writer.serializer.RocketMQSerializationSchema;
+import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector;
import org.apache.flink.util.FlinkRuntimeException;
import org.apache.rocketmq.client.producer.SendResult;
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/InnerConsumerImpl.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/InnerConsumerImpl.java
index c1c6907..8c79e0f 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/InnerConsumerImpl.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/InnerConsumerImpl.java
@@ -74,10 +74,8 @@ public class InnerConsumerImpl implements InnerConsumer {
String accessKey = configuration.getString(RocketMQSourceOptions.OPTIONAL_ACCESS_KEY);
String secretKey = configuration.getString(RocketMQSourceOptions.OPTIONAL_SECRET_KEY);
- boolean enableTrace =
- configuration.getBoolean(RocketMQSourceOptions.ENABLE_MESSAGE_TRACE);
- String traceTopic =
- configuration.getString(RocketMQSourceOptions.CUSTOMIZED_TRACE_TOPIC);
+ boolean enableTrace = configuration.getBoolean(RocketMQSourceOptions.ENABLE_MESSAGE_TRACE);
+ String traceTopic = configuration.getString(RocketMQSourceOptions.CUSTOMIZED_TRACE_TOPIC);
// Note: sync pull thread num may not enough
if (!StringUtils.isNullOrWhitespaceOnly(accessKey)
@@ -496,17 +494,13 @@ public class InnerConsumerImpl implements InnerConsumer {
@Override
public Map<MessageQueue, Long> minOffsets(Collection<MessageQueue> messageQueues) {
return fetchOffsets(
- messageQueues,
- mq -> innerConsumer.seekMinOffset(mq),
- "fetch min offset");
+ messageQueues, mq -> innerConsumer.seekMinOffset(mq), "fetch min offset");
}
@Override
public Map<MessageQueue, Long> maxOffsets(Collection<MessageQueue> messageQueues) {
return fetchOffsets(
- messageQueues,
- mq -> innerConsumer.seekMaxOffset(mq),
- "fetch max offset");
+ messageQueues, mq -> innerConsumer.seekMaxOffset(mq), "fetch max offset");
}
@Override
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSource.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSource.java
index 7f26d87..3430a04 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSource.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSource.java
@@ -48,7 +48,6 @@ import org.apache.flink.core.io.SimpleVersionedSerializer;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.util.UserCodeClassLoader;
-
import java.util.function.Supplier;
/** The Source implementation of RocketMQ. */
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSourceOptions.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSourceOptions.java
index e14667f..4683184 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSourceOptions.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/RocketMQSourceOptions.java
@@ -22,8 +22,8 @@ import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.connector.rocketmq.common.config.RocketMQConfigValidator;
import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions;
-import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig;
import org.apache.flink.connector.rocketmq.source.enumerator.allocate.AllocateStrategyFactory;
+import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig;
/** Includes config options of RocketMQ connector type. */
public class RocketMQSourceOptions extends RocketMQOptions {
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/enumerator/offset/OffsetsSelector.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/enumerator/offset/OffsetsSelector.java
index 68ceff2..c914f96 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/enumerator/offset/OffsetsSelector.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/enumerator/offset/OffsetsSelector.java
@@ -19,9 +19,9 @@
package org.apache.flink.connector.rocketmq.source.enumerator.offset;
import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.streaming.connectors.rocketmq.common.config.OffsetResetStrategy;
import org.apache.flink.connector.rocketmq.source.RocketMQSource;
import org.apache.flink.connector.rocketmq.source.split.RocketMQSourceSplit;
+import org.apache.flink.streaming.connectors.rocketmq.common.config.OffsetResetStrategy;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.common.message.MessageQueue;
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/RocketMQSourceFetcherManager.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/RocketMQSourceFetcherManager.java
index e04b771..a74b831 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/RocketMQSourceFetcherManager.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/RocketMQSourceFetcherManager.java
@@ -18,6 +18,7 @@
package org.apache.flink.connector.rocketmq.source.reader;
import org.apache.flink.annotation.Internal;
+import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
import org.apache.flink.connector.base.source.reader.SourceReaderBase;
import org.apache.flink.connector.base.source.reader.fetcher.SingleThreadFetcherManager;
@@ -25,7 +26,6 @@ import org.apache.flink.connector.base.source.reader.fetcher.SplitFetcher;
import org.apache.flink.connector.base.source.reader.fetcher.SplitFetcherTask;
import org.apache.flink.connector.base.source.reader.splitreader.SplitReader;
import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.rocketmq.source.split.RocketMQSourceSplit;
import org.apache.rocketmq.common.message.MessageQueue;
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/deserializer/RocketMQRowDeserializationSchema.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/deserializer/RocketMQRowDeserializationSchema.java
index ac9ef7d..65ac1f4 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/deserializer/RocketMQRowDeserializationSchema.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/connector/rocketmq/source/reader/deserializer/RocketMQRowDeserializationSchema.java
@@ -85,8 +85,7 @@ public class RocketMQRowDeserializationSchema implements RocketMQDeserialization
bytesMessage.setProperties(message.getProperties());
}
bytesMessage.setProperty("__topic__", message.getTopic());
- bytesMessage.setProperty(
- "__store_timestamp__", String.valueOf(message.getIngestionTime()));
+ bytesMessage.setProperty("__store_timestamp__", String.valueOf(message.getIngestionTime()));
bytesMessage.setProperty("__born_timestamp__", String.valueOf(message.getEventTime()));
bytesMessage.setProperty("__queue_id__", String.valueOf(message.getQueueId()));
bytesMessage.setProperty("__queue_offset__", String.valueOf(message.getQueueOffset()));
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSink.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSink.java
index d95c212..bc3ef5c 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSink.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSink.java
@@ -17,14 +17,14 @@
package org.apache.flink.streaming.connectors.rocketmq;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector;
-import org.apache.flink.streaming.connectors.rocketmq.common.util.MetricUtils;
import org.apache.flink.metrics.Meter;
import org.apache.flink.runtime.state.FunctionInitializationContext;
import org.apache.flink.runtime.state.FunctionSnapshotContext;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
+import org.apache.flink.streaming.connectors.rocketmq.common.selector.MessageQueueSelector;
+import org.apache.flink.streaming.connectors.rocketmq.common.util.MetricUtils;
import org.apache.flink.util.StringUtils;
import org.apache.commons.lang.Validate;
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceFunction.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceFunction.java
index c660335..42dc415 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceFunction.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceFunction.java
@@ -25,12 +25,6 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.connectors.rocketmq.common.config.OffsetResetStrategy;
-import org.apache.flink.streaming.connectors.rocketmq.common.config.StartupMode;
-import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema;
-import org.apache.flink.streaming.connectors.rocketmq.common.util.MetricUtils;
-import org.apache.flink.streaming.connectors.rocketmq.common.util.RetryUtil;
-import org.apache.flink.streaming.connectors.rocketmq.common.util.RocketMQUtils;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Meter;
import org.apache.flink.metrics.MeterView;
@@ -41,6 +35,12 @@ import org.apache.flink.runtime.state.FunctionSnapshotContext;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
+import org.apache.flink.streaming.connectors.rocketmq.common.config.OffsetResetStrategy;
+import org.apache.flink.streaming.connectors.rocketmq.common.config.StartupMode;
+import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema;
+import org.apache.flink.streaming.connectors.rocketmq.common.util.MetricUtils;
+import org.apache.flink.streaming.connectors.rocketmq.common.util.RetryUtil;
+import org.apache.flink.streaming.connectors.rocketmq.common.util.RocketMQUtils;
import org.apache.flink.util.Preconditions;
import org.apache.commons.collections.CollectionUtils;
@@ -70,7 +70,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
@@ -268,8 +267,7 @@ public class RocketMQSourceFunction<OUT> extends RichParallelSourceFunction<OUT>
}
}
- private void consumeMessages(
- MessageQueue mq, SourceContext<OUT> context, int pullBatchSize) {
+ private void consumeMessages(MessageQueue mq, SourceContext<OUT> context, int pullBatchSize) {
RetryUtil.call(
() -> {
pollAndEmitLoop(mq, context, pullBatchSize);
@@ -279,8 +277,7 @@ public class RocketMQSourceFunction<OUT> extends RichParallelSourceFunction<OUT>
runningChecker);
}
- private void pollAndEmitLoop(
- MessageQueue mq, SourceContext<OUT> context, int pullBatchSize) {
+ private void pollAndEmitLoop(MessageQueue mq, SourceContext<OUT> context, int pullBatchSize) {
while (runningChecker.isRunning()) {
try {
pollAndEmit(mq, context, pullBatchSize);
@@ -313,8 +310,7 @@ public class RocketMQSourceFunction<OUT> extends RichParallelSourceFunction<OUT>
}
if (!found) {
- RetryUtil.waitForMs(
- RocketMQConfig.DEFAULT_CONSUMER_DELAY_WHEN_MESSAGE_NOT_FOUND);
+ RetryUtil.waitForMs(RocketMQConfig.DEFAULT_CONSUMER_DELAY_WHEN_MESSAGE_NOT_FOUND);
}
}
@@ -322,9 +318,7 @@ public class RocketMQSourceFunction<OUT> extends RichParallelSourceFunction<OUT>
long fetchTime = System.currentTimeMillis();
for (MessageExt msg : messages) {
byte[] key =
- msg.getKeys() != null
- ? msg.getKeys().getBytes(StandardCharsets.UTF_8)
- : null;
+ msg.getKeys() != null ? msg.getKeys().getBytes(StandardCharsets.UTF_8) : null;
byte[] value = msg.getBody();
OUT data = schema.deserializeKeyAndValue(key, value);
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSink.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSink.java
index bb05b67..1a4bb04 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSink.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSink.java
@@ -21,9 +21,9 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.RowTypeInfo;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions;
-import org.apache.flink.connector.rocketmq.sink.RocketMQSinkOptions;
import org.apache.flink.connector.rocketmq.sink.RocketMQSink;
import org.apache.flink.connector.rocketmq.sink.RocketMQSinkBuilder;
+import org.apache.flink.connector.rocketmq.sink.RocketMQSinkOptions;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.ChangelogMode;
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactory.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactory.java
index 4807c4d..c5754c7 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactory.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactory.java
@@ -20,9 +20,9 @@ package org.apache.flink.streaming.connectors.rocketmq.table;
import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig;
import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions;
import org.apache.flink.connector.rocketmq.source.RocketMQSourceOptions;
+import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.descriptors.DescriptorProperties;
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSerializationSchema.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSerializationSchema.java
index 5cc0755..3a3418d 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSerializationSchema.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSerializationSchema.java
@@ -26,8 +26,8 @@ import org.apache.rocketmq.common.message.Message;
/**
* A {@link RocketMQSerializationSchema} adapter that bridges the Table/SQL {@link RowData} to
- * RocketMQ {@link Message} conversion. Delegates to {@link RocketMQRowDataConverter} for the
- * actual conversion logic.
+ * RocketMQ {@link Message} conversion. Delegates to {@link RocketMQRowDataConverter} for the actual
+ * conversion logic.
*/
public class RocketMQRowDataSerializationSchema implements RocketMQSerializationSchema<RowData> {
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSink.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSink.java
index 138d79c..cc8ee35 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSink.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQRowDataSink.java
@@ -16,8 +16,8 @@ package org.apache.flink.streaming.connectors.rocketmq.table;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.connectors.rocketmq.RocketMQSink;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.streaming.connectors.rocketmq.RocketMQSink;
import org.apache.flink.table.data.RowData;
import org.apache.rocketmq.common.message.Message;
diff --git a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSource.java b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSource.java
index 32afa6e..6d12961 100644
--- a/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSource.java
+++ b/flink-connector-rocketmq/src/main/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSource.java
@@ -19,10 +19,6 @@ package org.apache.flink.streaming.connectors.rocketmq.table;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.connector.rocketmq.common.config.RocketMQOptions;
-import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig;
-import org.apache.flink.streaming.connectors.rocketmq.RocketMQSourceFunction;
-import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema;
-import org.apache.flink.streaming.connectors.rocketmq.common.serialization.RowKeyValueDeserializationSchema;
import org.apache.flink.connector.rocketmq.source.RocketMQSource;
import org.apache.flink.connector.rocketmq.source.RocketMQSourceBuilder;
import org.apache.flink.connector.rocketmq.source.RocketMQSourceOptions;
@@ -32,6 +28,10 @@ import org.apache.flink.connector.rocketmq.source.reader.deserializer.BytesMessa
import org.apache.flink.connector.rocketmq.source.reader.deserializer.RocketMQDeserializationSchema;
import org.apache.flink.connector.rocketmq.source.reader.deserializer.RocketMQRowDeserializationSchema;
import org.apache.flink.connector.rocketmq.source.reader.deserializer.RowDeserializationSchema.MetadataConverter;
+import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig;
+import org.apache.flink.streaming.connectors.rocketmq.RocketMQSourceFunction;
+import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema;
+import org.apache.flink.streaming.connectors.rocketmq.common.serialization.RowKeyValueDeserializationSchema;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.ChangelogMode;
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/catalog/RocketMQCatalogTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/catalog/RocketMQCatalogTest.java
index e8f2c1f..1611b02 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/catalog/RocketMQCatalogTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/catalog/RocketMQCatalogTest.java
@@ -23,15 +23,9 @@ import org.apache.flink.table.catalog.CatalogDatabase;
import org.apache.flink.table.catalog.CatalogPartition;
import org.apache.flink.table.catalog.CatalogPartitionSpec;
import org.apache.flink.table.catalog.ObjectPath;
-import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
-import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
-import org.apache.flink.table.catalog.exceptions.FunctionAlreadyExistException;
import org.apache.flink.table.catalog.exceptions.FunctionNotExistException;
-import org.apache.flink.table.catalog.exceptions.PartitionAlreadyExistsException;
import org.apache.flink.table.catalog.exceptions.PartitionNotExistException;
-import org.apache.flink.table.catalog.exceptions.PartitionSpecInvalidException;
-import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
import org.apache.flink.table.catalog.exceptions.TableNotExistException;
import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException;
import org.apache.flink.table.catalog.stats.CatalogColumnStatistics;
@@ -215,8 +209,7 @@ public class RocketMQCatalogTest {
@Test
public void testDropTable() {
assertThrows(
- UnsupportedOperationException.class,
- () -> rocketMQCatalog.dropTable(null, false));
+ UnsupportedOperationException.class, () -> rocketMQCatalog.dropTable(null, false));
}
@Test
@@ -239,8 +232,7 @@ public class RocketMQCatalogTest {
@Test
public void testFunctionExists() {
assertThrows(
- UnsupportedOperationException.class,
- () -> rocketMQCatalog.functionExists(null));
+ UnsupportedOperationException.class, () -> rocketMQCatalog.functionExists(null));
}
@Test
@@ -274,8 +266,7 @@ public class RocketMQCatalogTest {
@Test
public void testListViews() {
assertThrows(
- UnsupportedOperationException.class,
- () -> rocketMQCatalog.listViews("default"));
+ UnsupportedOperationException.class, () -> rocketMQCatalog.listViews("default"));
}
@Test
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/ConnectorIntegrationTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/ConnectorIntegrationTest.java
index 27c4f3a..0b973e6 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/ConnectorIntegrationTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/ConnectorIntegrationTest.java
@@ -43,9 +43,7 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import static org.apache.flink.connector.rocketmq.example.ConnectorConfig.ACCESS_KEY;
import static org.apache.flink.connector.rocketmq.example.ConnectorConfig.CONSUMER_GROUP;
@@ -122,8 +120,8 @@ public class ConnectorIntegrationTest {
// ---- Step 1: Start Flink Source → Sink pipeline ----
private Thread startFlinkPipeline() throws Exception {
- System.out.println("[Step 1] Starting Flink pipeline: "
- + SOURCE_TOPIC_1 + " → " + SINK_TOPIC_1);
+ System.out.println(
+ "[Step 1] Starting Flink pipeline: " + SOURCE_TOPIC_1 + " → " + SINK_TOPIC_1);
StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(1);
@@ -138,8 +136,7 @@ public class ConnectorIntegrationTest {
.setMinOffsets(OffsetsSelector.latest())
.setConfig(RocketMQOptions.OPTIONAL_ACCESS_KEY, ACCESS_KEY)
.setConfig(RocketMQOptions.OPTIONAL_SECRET_KEY, SECRET_KEY)
- .setConfig(
- RocketMQOptions.OPTIONAL_ACCESS_CHANNEL, AccessChannel.CLOUD)
+ .setConfig(RocketMQOptions.OPTIONAL_ACCESS_CHANNEL, AccessChannel.CLOUD)
.setDeserializer(new StringBodyDeserializationSchema())
.build();
@@ -150,8 +147,7 @@ public class ConnectorIntegrationTest {
.setGroupId(PRODUCER_GROUP)
.setConfig(RocketMQOptions.OPTIONAL_ACCESS_KEY, ACCESS_KEY)
.setConfig(RocketMQOptions.OPTIONAL_SECRET_KEY, SECRET_KEY)
- .setConfig(
- RocketMQOptions.OPTIONAL_ACCESS_CHANNEL, AccessChannel.CLOUD)
+ .setConfig(RocketMQOptions.OPTIONAL_ACCESS_CHANNEL, AccessChannel.CLOUD)
.setDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE)
.setSerializer(
(element, context, timestamp) ->
@@ -215,8 +211,7 @@ public class ConnectorIntegrationTest {
producer.shutdown();
}
- System.out.printf(
- "[Step 2] Done: %d/%d messages sent%n%n", successCount, MESSAGE_COUNT);
+ System.out.printf("[Step 2] Done: %d/%d messages sent%n%n", successCount, MESSAGE_COUNT);
if (successCount < MESSAGE_COUNT) {
throw new RuntimeException(
"Failed to produce all messages: " + successCount + "/" + MESSAGE_COUNT);
@@ -265,7 +260,9 @@ public class ConnectorIntegrationTest {
receivedMessages.size(), MESSAGE_COUNT);
assertTrue(
- "Expected at least " + MESSAGE_COUNT + " messages in sink, got "
+ "Expected at least "
+ + MESSAGE_COUNT
+ + " messages in sink, got "
+ receivedMessages.size(),
receivedMessages.size() >= MESSAGE_COUNT);
}
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/SqlIntegrationTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/SqlIntegrationTest.java
index 9d78fe6..f00336b 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/SqlIntegrationTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/SqlIntegrationTest.java
@@ -128,7 +128,8 @@ public class SqlIntegrationTest {
}
private void produceMessages() throws Exception {
- System.out.println("[Step 2] Producing " + MESSAGE_COUNT + " messages to " + SOURCE_TOPIC_2);
+ System.out.println(
+ "[Step 2] Producing " + MESSAGE_COUNT + " messages to " + SOURCE_TOPIC_2);
DefaultMQProducer producer =
new DefaultMQProducer(PRODUCER_GROUP, getAclRpcHook(), true, null);
producer.setNamesrvAddr(ENDPOINTS);
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/TransactionSinkIntegrationTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/TransactionSinkIntegrationTest.java
index 7bbd015..266ec75 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/TransactionSinkIntegrationTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/example/TransactionSinkIntegrationTest.java
@@ -54,8 +54,8 @@ import static org.junit.Assert.assertTrue;
* <li>Record the current max offsets of the transaction topic
* <li>Run a bounded Flink job writing messages with EXACTLY_ONCE; checkpointing triggers the
* two-phase commit (prepareCommit → Committer.commit → endTransaction COMMIT)
- * <li>Consume the topic from the recorded offsets and verify every message became visible,
- * which only happens after a successful transaction commit
+ * <li>Consume the topic from the recorded offsets and verify every message became visible, which
+ * only happens after a successful transaction commit
* </ol>
*
* <p>Skipped automatically when environment variables are not set. The topic must be a
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/sink/InnerProducerImplTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/sink/InnerProducerImplTest.java
index 7de3db7..84a216e 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/sink/InnerProducerImplTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/connector/rocketmq/sink/InnerProducerImplTest.java
@@ -29,7 +29,8 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Tests for verifying that configuration parameters are applied to the RocketMQ producer. */
class InnerProducerImplTest {
- private TransactionMQProducer getProducerViaReflection(InnerProducerImpl impl) throws Exception {
+ private TransactionMQProducer getProducerViaReflection(InnerProducerImpl impl)
+ throws Exception {
Field field = InnerProducerImpl.class.getDeclaredField("producer");
field.setAccessible(true);
return (TransactionMQProducer) field.get(impl);
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceTest.java
index 1e60937..5f6c7a9 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/RocketMQSourceTest.java
@@ -18,10 +18,10 @@
package org.apache.flink.streaming.connectors.rocketmq;
+import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext;
import org.apache.flink.streaming.connectors.rocketmq.common.serialization.KeyValueDeserializationSchema;
import org.apache.flink.streaming.connectors.rocketmq.common.serialization.SimpleKeyValueDeserializationSchema;
import org.apache.flink.streaming.connectors.rocketmq.common.util.TestUtils;
-import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext;
import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer;
import org.apache.rocketmq.client.consumer.PullResult;
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/example/LegacyConnectorExample.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/example/LegacyConnectorExample.java
index c65fa1e..684680a 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/example/LegacyConnectorExample.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/example/LegacyConnectorExample.java
@@ -19,18 +19,18 @@ package org.apache.flink.streaming.connectors.rocketmq.example;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig;
-import org.apache.flink.streaming.connectors.rocketmq.RocketMQSink;
-import org.apache.flink.streaming.connectors.rocketmq.RocketMQSourceFunction;
-import org.apache.flink.streaming.connectors.rocketmq.common.serialization.SimpleTupleDeserializationSchema;
-import org.apache.flink.streaming.connectors.rocketmq.function.SinkMapFunction;
-import org.apache.flink.streaming.connectors.rocketmq.function.SourceMapFunction;
import org.apache.flink.runtime.state.memory.MemoryStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.connectors.rocketmq.RocketMQConfig;
+import org.apache.flink.streaming.connectors.rocketmq.RocketMQSink;
+import org.apache.flink.streaming.connectors.rocketmq.RocketMQSourceFunction;
+import org.apache.flink.streaming.connectors.rocketmq.common.serialization.SimpleTupleDeserializationSchema;
+import org.apache.flink.streaming.connectors.rocketmq.function.SinkMapFunction;
+import org.apache.flink.streaming.connectors.rocketmq.function.SourceMapFunction;
import org.apache.rocketmq.client.AccessChannel;
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactoryOffsetTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactoryOffsetTest.java
index c9db6f1..f0ce69a 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactoryOffsetTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQDynamicTableSourceFactoryOffsetTest.java
@@ -35,7 +35,6 @@ import java.util.HashMap;
import java.util.Map;
import static org.apache.flink.table.api.DataTypes.BIGINT;
-import static org.apache.flink.table.api.DataTypes.STRING;
import static org.assertj.core.api.Assertions.assertThat;
/**
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceFilterTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceFilterTest.java
index bcf163b..13a5c7b 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceFilterTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceFilterTest.java
@@ -66,8 +66,7 @@ class RocketMQScanTableSourceFilterTest {
@Test
void sqlFilterShouldBePreservedOnCopy() {
DescriptorProperties props = new DescriptorProperties();
- TableSchema schema =
- TableSchema.builder().field("id", DataTypes.BIGINT()).build();
+ TableSchema schema = TableSchema.builder().field("id", DataTypes.BIGINT()).build();
RocketMQScanTableSource source =
new RocketMQScanTableSource(
@@ -97,8 +96,7 @@ class RocketMQScanTableSourceFilterTest {
void wildcardTagShouldMatchAllMessages() {
// "*" is the default tag and should not filter any messages
DescriptorProperties props = new DescriptorProperties();
- TableSchema schema =
- TableSchema.builder().field("id", DataTypes.BIGINT()).build();
+ TableSchema schema = TableSchema.builder().field("id", DataTypes.BIGINT()).build();
RocketMQScanTableSource source =
new RocketMQScanTableSource(
diff --git a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceTest.java b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceTest.java
index 0000f9f..0fa158b 100644
--- a/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceTest.java
+++ b/flink-connector-rocketmq/src/test/java/org/apache/flink/streaming/connectors/rocketmq/table/RocketMQScanTableSourceTest.java
@@ -17,7 +17,6 @@
package org.apache.flink.streaming.connectors.rocketmq.table;
-import org.apache.flink.connector.rocketmq.source.RocketMQSource;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.source.ScanTableSource;
diff --git a/flink-sql-connector-rocketmq-grpc/pom.xml b/flink-sql-connector-rocketmq-grpc/pom.xml
new file mode 100644
index 0000000..963169a
--- /dev/null
+++ b/flink-sql-connector-rocketmq-grpc/pom.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-rocketmq-parent</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>flink-sql-connector-rocketmq-grpc</artifactId>
+ <name>Flink : Connectors : RocketMQ gRPC : SQL</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <japicmp.skip>true</japicmp.skip>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-rocketmq-grpc</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-connector-base</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Test dependencies -->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>shade-flink</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <includes>
+ <include>org.apache.flink:flink-connector-rocketmq-grpc</include>
+ <include>org.apache.rocketmq:rocketmq-client-java</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.apache.rocketmq:*</artifact>
+ <excludes>
+ <exclude>LICENSE</exclude>
+ <exclude>NOTICE</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.rocketmq</pattern>
+ <shadedPattern>org.apache.flink.rocketmq.grpc.shaded.org.apache.rocketmq</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>analyze-deps</id>
+ <goals>
+ <goal>analyze</goal>
+ </goals>
+ <phase>verify</phase>
+ <configuration>
+ <ignoredUnusedDeclaredDependencies>
+ org.apache.flink:flink-connector-rocketmq-grpc
+ </ignoredUnusedDeclaredDependencies>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/pom.xml b/pom.xml
index c92cf4f..7570a25 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,7 +73,9 @@ under the License.
<modules>
<module>flink-connector-rocketmq</module>
+ <module>flink-connector-rocketmq-grpc</module>
<module>flink-sql-connector-rocketmq</module>
+ <module>flink-sql-connector-rocketmq-grpc</module>
</modules>
<properties>