You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by sh...@apache.org on 2023/05/19 07:19:17 UTC
[rocketmq-eventbridge] branch runtimer updated: feature:update runner json file, rule transfer biz optimize, commit biz (#102)
This is an automated email from the ASF dual-hosted git repository.
shenlin pushed a commit to branch runtimer
in repository https://gitbox.apache.org/repos/asf/rocketmq-eventbridge.git
The following commit(s) were added to refs/heads/runtimer by this push:
new 4bbe4e3 feature:update runner json file, rule transfer biz optimize, commit biz (#102)
4bbe4e3 is described below
commit 4bbe4e349d9a8958f39be6e465a2b0d78f26f67c
Author: Artisan <Xi...@outlook.com>
AuthorDate: Fri May 19 15:19:10 2023 +0800
feature:update runner json file, rule transfer biz optimize, commit biz (#102)
* update subscriber and listener
* init 2nd edition for Runtimer
* rename few class and add propertis config
* optimize concurrency biz
* optimize jsonFile formate
* update reflect
* completed config style
* completed test
* update config,transfer,pusher
* updateFewFileNaming
* rename Pusher to Trigger, Runtimer to runtime, and optimize few architect
* updateRocketMQSubscriberFile
---
adapter/persistence/pom.xml | 2 +-
adapter/pom.xml | 2 +-
adapter/rpc/pom.xml | 4 +-
.../impl/runtime/RuntimeTargetRunnerAPIImpl.java | 2 +-
adapter/runtime/README.md | 68 +++++
adapter/{runtimer => runtime}/pom.xml | 2 +-
.../eventbridge/adapter/runtime/Runtime.java} | 54 ++--
.../adapter/runtime}/boot/EventBusListener.java | 21 +-
.../adapter/runtime}/boot/EventRuleTransfer.java | 83 +++---
.../adapter/runtime/boot/EventTargetTrigger.java} | 57 ++--
.../runtime/boot/common/CirculatorContext.java | 294 +++++++++++++++++++++
.../runtime/boot/common}/OffsetManager.java | 13 +-
.../runtime/boot/common}/TargetRunnerListener.java | 4 +-
.../runtime}/boot/listener/EventSubscriber.java | 20 +-
.../runtime}/boot/transfer/TransformEngine.java | 24 +-
.../runtime/boot/trigger/TriggerTaskContext.java} | 44 +--
.../runtime}/common/FileAndPropertyUtil.java | 2 +-
.../runtime}/common/FilePathConfigUtil.java | 2 +-
.../adapter/runtime}/common/LoggerName.java | 4 +-
.../adapter/runtime}/common/QueueState.java | 2 +-
.../adapter/runtime/common/RuntimeState.java} | 6 +-
.../adapter/runtime}/common/ServiceThread.java | 3 +-
.../runtime/common/entity/SubscribeRunnerKeys.java | 58 ++++
.../runtime}/common/entity/TargetKeyValue.java | 2 +-
.../runtime}/common/entity/TargetRunnerConfig.java | 17 +-
.../runtime}/common/enums/ConfigModeEnum.java | 2 +-
.../runtime}/common/enums/RefreshTypeEnum.java | 2 +-
.../runtime/common/enums/StorageModeEnum.java} | 22 +-
.../adapter/runtime}/common/plugin/Plugin.java | 4 +-
.../runtime}/common/plugin/PluginClassLoader.java | 2 +-
.../runtime}/common/plugin/PluginUtils.java | 2 +-
.../runtime}/common/plugin/PluginWrapper.java | 2 +-
.../common/store/FileBaseKeyValueStore.java | 6 +-
.../runtime}/common/store/KeyValueStore.java | 2 +-
.../common/store/MemoryBasedKeyValueStore.java | 4 +-
.../runtime/config/RuntimeConfigDefine.java} | 29 +-
.../runtime/config/RuntimeConfigProps.java} | 14 +-
.../runtime/config/RuntimeConfiguration.java} | 14 +-
.../adapter/runtime}/config/TraceConfig.java | 2 +-
.../adapter/runtime}/converter/JsonConverter.java | 4 +-
.../adapter/runtime}/converter/ListConverter.java | 4 +-
.../runtime}/converter/RecordOffsetConverter.java | 4 +-
.../runtime}/error/DeadLetterQueueService.java | 2 +-
.../adapter/runtime}/error/ErrorHandler.java | 23 +-
.../AbstractTargetRunnerConfigObserver.java | 23 +-
.../service/TargetRunnerConfigObserver.java | 17 +-
.../TargetRunnerConfigOnControllerObserver.java | 8 +-
.../service/TargetRunnerConfigOnDBObserver.java | 4 +-
.../service/TargetRunnerConfigOnFileObserver.java | 8 +-
.../runtime}/trace/JaegerTraceStrategy.java | 2 +-
.../adapter/runtime}/trace/TraceConstants.java | 2 +-
.../adapter/runtime}/trace/TraceFactory.java | 2 +-
.../src/main/resources/runtime.properties} | 2 +-
.../src/main/resources/target-runner.json | 4 +-
.../TargetRunnerConfigOnFileObserverTest.java | 6 +-
.../runtimer/service/TargetRunnerConfigUtil.java | 2 +-
.../runtimer/service/TestConstants.java | 0
.../ConsumerSlowerPIDControllerTest.java | 0
.../backpressure/MockEventBusListenerTest.java | 2 +-
.../service/backpressure/PIDContextTest.java | 0
.../service/backpressure/PIDController.java | 0
.../ProductSlowerPIDControllerTest.java | 0
.../runtimer/service/backpressure/PusherTest.java | 2 +-
.../service/backpressure/RateEstimatorTest.java | 2 +-
.../service/backpressure/TransformTest.java | 2 +-
adapter/runtimer/README.md | 42 ---
.../runtimer/boot/listener/CirculatorContext.java | 228 ----------------
.../boot/listener/TargetRunnerContext.java | 46 ----
.../runtimer/config/RuntimerConfigDefine.java | 108 --------
adapter/storage/pom.xml | 2 +-
.../rocketmq/impl/RocketMQEventDataRepository.java | 1 +
.../rocketmq/runtimer/RocketMQEventSubscriber.java | 130 +++++----
.../domain/storage/EventDataRepository.java | 7 +
infrastructure/README-CN.md | 16 ++
{adapter => infrastructure}/pom.xml | 19 +-
.../infrastructure/metric/MonitorFactory.java | 4 +-
.../infrastructure}/trace/TraceException.java | 2 +-
.../infrastructure}/trace/TraceStrategy.java | 2 +-
.../infrastructure}/trace/ZipkinTraceStrategy.java | 2 +-
pom.xml | 8 +-
start/pom.xml | 6 +-
start/src/main/resources/application.properties | 11 +-
82 files changed, 856 insertions(+), 800 deletions(-)
diff --git a/adapter/persistence/pom.xml b/adapter/persistence/pom.xml
index fdbb7a3..dd46427 100644
--- a/adapter/persistence/pom.xml
+++ b/adapter/persistence/pom.xml
@@ -33,7 +33,7 @@
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-eventbridge-adapter-runtimer</artifactId>
+ <artifactId>rocketmq-eventbridge-adapter-runtime</artifactId>
</dependency>
<!-- Framework -->
<dependency>
diff --git a/adapter/pom.xml b/adapter/pom.xml
index ab600bb..887be3e 100644
--- a/adapter/pom.xml
+++ b/adapter/pom.xml
@@ -25,7 +25,7 @@
<module>persistence</module>
<module>api</module>
<module>rpc</module>
- <module>runtimer</module>
+ <module>runtime</module>
<module>storage</module>
</modules>
diff --git a/adapter/rpc/pom.xml b/adapter/rpc/pom.xml
index 7aaf521..01cd293 100644
--- a/adapter/rpc/pom.xml
+++ b/adapter/rpc/pom.xml
@@ -30,7 +30,7 @@
<!-- Project Modules -->
<dependency>
<groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-eventbridge-adapter-runtimer</artifactId>
+ <artifactId>rocketmq-eventbridge-adapter-runtime</artifactId>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
@@ -38,7 +38,7 @@
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-eventbridge-adapter-runtimer</artifactId>
+ <artifactId>rocketmq-eventbridge-adapter-runtime</artifactId>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
diff --git a/adapter/rpc/src/main/java/org/apache/rocketmq/eventbridge/adapter/rpc/impl/runtime/RuntimeTargetRunnerAPIImpl.java b/adapter/rpc/src/main/java/org/apache/rocketmq/eventbridge/adapter/rpc/impl/runtime/RuntimeTargetRunnerAPIImpl.java
index ee66625..77c6bc9 100644
--- a/adapter/rpc/src/main/java/org/apache/rocketmq/eventbridge/adapter/rpc/impl/runtime/RuntimeTargetRunnerAPIImpl.java
+++ b/adapter/rpc/src/main/java/org/apache/rocketmq/eventbridge/adapter/rpc/impl/runtime/RuntimeTargetRunnerAPIImpl.java
@@ -23,7 +23,7 @@ import com.google.gson.Gson;
import java.util.List;
import java.util.Map;
import org.apache.rocketmq.eventbridge.adapter.rpc.impl.connect.RocketMQConverter;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
import org.apache.rocketmq.eventbridge.domain.common.enums.EventTargetStatusEnum;
import org.apache.rocketmq.eventbridge.domain.model.Component;
import org.apache.rocketmq.eventbridge.domain.model.run.RunOptions;
diff --git a/adapter/runtime/README.md b/adapter/runtime/README.md
new file mode 100644
index 0000000..5a64318
--- /dev/null
+++ b/adapter/runtime/README.md
@@ -0,0 +1,68 @@
+架构
+事件网关将接收用户发送到 EventBridge 的事件,并将事件传递到内置存储 RocketMQ。
+当用户将规则配置为订阅事件总线上的事件时,推送运行时将侦听与规则关联的总线,一旦总线上收到事件,它将事件推送到规则指定的目标。
+如果在规则上配置了筛选和转换,则事件将在推送前进行预处理。
+
+推送运行时如何侦听事件总线
+这里有两种方法:
+
+方案一-独立模式:每个规则单独创建一个MQ订阅,监控BUS上是否存在与规则关联的事件;在此模式下,每个规则订阅都是一个独立的任务。
+
+方案 2 - 共享模式:推送运行时计时器中的统一组件监控与分配给当前运行时的所有规则关联的 BUS;如果监控总线上发生事件,则会将事件分发到指定的规则进行过滤和转换。最后,它通过接收器连接器被推到目标侧;
+
+由于事件规则与流式处理方案不同,因此大多数情况下,规则监视的总线上可能没有事件;为了降低监控 MQ 的成本,推荐方案 2:推送运行时中的统一组件监控分配给当前运行时计时器的所有规则并共享资源。
+
+推送运行时管理器
+问题分析:
+
+如果每个正在运行的推送运行时需要加载所有规则,当规则越来越多时,推送运行时无法水平扩展,将成为瓶颈。
+
+方案:
+推送运行时集群只需要加载分配给它的规则,并根据规则的定义监听指定的 BUS。一旦总线上发生事件,就会根据过滤规则过滤事件,推送到目标终端目标。
+推送运行时管理器的职责:
+
+所有推送运行时生命周期都由推送运行时管理器管理。
+如何将规则分配给推送运行时由推送运行时管理器管理。
+
+接口设计/更改
+方法签名更改
+方法行为更改
+CLI 命令更改
+日志格式或内容更改
+问题分析:
+EventBridge 是一个开放的架构。事件网关接收的事件可能不会存储在 RocketMQ 中。因此,在设计推送运行时,需要支持对不同 MQ 的监控。因此,我们需要在这里定义一组标准 SPI,并实现不同的 MQ 监控组件;
+推送运行时侦听总线上的事件后,会对事件进行过滤和转换,最后将事件推送到规则中指定的目的地。在推送之前,事件将被过滤和转换。这些过程需要定义SPI以供以后扩展,并促进生态合作伙伴的共建。
+方案: 目前,OpenMessaging Connect API 已经定义了完整的 SPI,并具有一定的生态。因此,我们考虑重用OpenMessaging Connect API作为我们的标准SPI。
+
+兼容性、弃用和迁移计划
+此提案不需要修改 API 规范。这只会向事件桥添加新的运行时组件。
+
+实施大纲
+我们将此提案分为几个任务:
+
+任务 1:推送运行时实现
+
+监视当前运行时需要处理的事件规则,以及与事件规则关联的总线。
+通过长池监控 MQ 组件。当检测到新事件时,它会立即将事件写入本地缓冲区,并且监视组件将继续监视后续事件;(事件推送不会阻塞监听线程)
+轮询组件从缓冲区获取事件,并根据上下文信息 (规则) 筛选和转换事件,然后将其发送到接收器连接器。
+Put 可以是异步操作或同步操作。
+运行时每次提交站点之前,都会调用 PreCommit 获取当前事件成功提交的偏移量,然后通过 MQ 监控组件提交站点。
+图像
+
+任务 2:推送运行时管理器实现
+
+推送运行时管理器如何将规则分配给推送运行时
+通过现有的event_target_runner表将事件规则分配给指定的推送运行时。
+
+添加event_runtimer表以管理所有运行时节点
+
+推送管理器根据指标信息(例如每个运行时计时器的负载(例如 CPU、内存、推送延迟时间和线程池的剩余大小)将运行时分配给规则目标。在早期阶段,可以根据分配给每个运行时的规则目标数量均匀分布;
+
+推送运行时计时器如何获取分配给它的规则目标
+可以提前创建运行时,然后在event_runtimer表中进行配置。如果底层基础设施支持自动创建容器的功能,您可以通过 PushManager 调用基础设施服务,动态创建运行时,并在event_runtimer表中自动注册它们。
+
+所有创建的运行时都有一个系统变量:runtimer-name,用于指定当前运行时的信息。当运行计时器启动时,它可以根据运行时程序名称获取分配给它的规则目标。
+
+此外,规则目标信息将在运行过程中更改。此时,PushManger 需要将最新更改的规则-目标信息通知相应的运行计时器。收到通知后,运行计时器将重新加载最新的规则目标。
+
+在开发和设计推送运行时时,请依赖Openmessaging Connect的API。
\ No newline at end of file
diff --git a/adapter/runtimer/pom.xml b/adapter/runtime/pom.xml
similarity index 98%
rename from adapter/runtimer/pom.xml
rename to adapter/runtime/pom.xml
index 0c7a5cc..a3f501f 100644
--- a/adapter/runtimer/pom.xml
+++ b/adapter/runtime/pom.xml
@@ -20,7 +20,7 @@
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>rocketmq-eventbridge-adapter-runtimer</artifactId>
+ <artifactId>rocketmq-eventbridge-adapter-runtime</artifactId>
<version>1.0.0</version>
<properties>
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/Runtimer.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/Runtime.java
similarity index 53%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/Runtimer.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/Runtime.java
index 0cf382b..e5bd487 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/Runtimer.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/Runtime.java
@@ -15,35 +15,36 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer;
+package org.apache.rocketmq.eventbridge.adapter.runtime;
-import java.util.concurrent.atomic.AtomicReference;
-import javax.annotation.PostConstruct;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.EventBusListener;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.EventRuleTransfer;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.EventTargetPusher;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.OffsetManager;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.CirculatorContext;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.EventSubscriber;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.RuntimerState;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.error.ErrorHandler;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.service.TargetRunnerConfigObserver;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.EventBusListener;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.EventRuleTransfer;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.EventTargetTrigger;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.CirculatorContext;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.OffsetManager;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.listener.EventSubscriber;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.RuntimeState;
+import org.apache.rocketmq.eventbridge.adapter.runtime.error.ErrorHandler;
+import org.apache.rocketmq.eventbridge.adapter.runtime.service.TargetRunnerConfigObserver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+import javax.annotation.PostConstruct;
+import java.util.concurrent.atomic.AtomicReference;
+
/**
- * event bridge runtimer
+ * event bridge runtime
*
* @author artisan
*/
@Component
-public class Runtimer {
+public class Runtime {
- private static final Logger logger = LoggerFactory.getLogger(Runtimer.class);
+ private static final Logger logger = LoggerFactory.getLogger(Runtime.class);
- private AtomicReference<RuntimerState> runtimerState;
+ private AtomicReference<RuntimeState> runtimerState;
@Autowired
private CirculatorContext circulatorContext;
@@ -56,33 +57,20 @@ public class Runtimer {
@Autowired
private ErrorHandler errorHandler;
- public Runtimer(
- CirculatorContext circulatorContext,
- TargetRunnerConfigObserver runnerConfigObserver,
- OffsetManager offsetManager,
- EventSubscriber eventSubscriber,
- ErrorHandler errorHandler) {
- this.circulatorContext = circulatorContext;
- this.runnerConfigObserver = runnerConfigObserver;
- this.offsetManager = offsetManager;
- this.eventSubscriber = eventSubscriber;
- this.errorHandler = errorHandler;
- }
-
@PostConstruct
public void initAndStart() {
- logger.info("Start init runtimer.");
- circulatorContext.initListenerMetadata(runnerConfigObserver.getTargetRunnerConfig());
+ logger.info("Start init runtime.");
+ circulatorContext.initCirculatorContext(runnerConfigObserver.getTargetRunnerConfig());
runnerConfigObserver.registerListener(circulatorContext);
runnerConfigObserver.registerListener(eventSubscriber);
new EventBusListener(circulatorContext, eventSubscriber, errorHandler).start();
new EventRuleTransfer(circulatorContext, offsetManager, errorHandler).start();
- new EventTargetPusher(circulatorContext, offsetManager, errorHandler).start();
+ new EventTargetTrigger(circulatorContext, offsetManager, errorHandler).start();
startRuntimer();
}
public void startRuntimer() {
- runtimerState = new AtomicReference<>(RuntimerState.START);
+ runtimerState = new AtomicReference<>(RuntimeState.START);
}
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventBusListener.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventBusListener.java
similarity index 73%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventBusListener.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventBusListener.java
index d735e9a..e6f06c2 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventBusListener.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventBusListener.java
@@ -15,15 +15,16 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot;
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot;
+import com.google.common.collect.Lists;
import io.openmessaging.connector.api.data.ConnectRecord;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.CirculatorContext;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.EventSubscriber;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.ServiceThread;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.error.ErrorHandler;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.CirculatorContext;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.listener.EventSubscriber;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.ServiceThread;
+import org.apache.rocketmq.eventbridge.adapter.runtime.error.ErrorHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,9 +38,7 @@ public class EventBusListener extends ServiceThread {
private static final Logger logger = LoggerFactory.getLogger(EventBusListener.class);
private final CirculatorContext circulatorContext;
-
private final EventSubscriber eventSubscriber;
-
private final ErrorHandler errorHandler;
public EventBusListener(CirculatorContext circulatorContext, EventSubscriber eventSubscriber,
@@ -52,15 +51,17 @@ public class EventBusListener extends ServiceThread {
@Override
public void run() {
while (!stopped) {
+ List<ConnectRecord> pullRecordList = Lists.newArrayList();
try {
- List<ConnectRecord> recordList = eventSubscriber.pull();
- if (CollectionUtils.isEmpty(recordList)) {
+ pullRecordList = eventSubscriber.pull();
+ if (CollectionUtils.isEmpty(pullRecordList)) {
this.waitForRunning(1000);
continue;
}
- circulatorContext.offerEventRecords(recordList);
+ circulatorContext.offerEventRecords(pullRecordList);
} catch (Exception exception) {
logger.error(getServiceName() + " - event bus pull record exception, stackTrace - ", exception);
+ pullRecordList.forEach(pullRecord -> errorHandler.handle(pullRecord, exception));
}
}
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventRuleTransfer.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventRuleTransfer.java
similarity index 52%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventRuleTransfer.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventRuleTransfer.java
index a094d07..8612733 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventRuleTransfer.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventRuleTransfer.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot;
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.Lists;
@@ -23,17 +23,15 @@ import io.openmessaging.connector.api.data.ConnectRecord;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
-import org.apache.commons.collections.CollectionUtils;
+
import org.apache.commons.collections.MapUtils;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.CirculatorContext;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.transfer.TransformEngine;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.ServiceThread;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.config.RuntimerConfigDefine;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.error.ErrorHandler;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.OffsetManager;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.CirculatorContext;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.transfer.TransformEngine;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.ServiceThread;
+import org.apache.rocketmq.eventbridge.adapter.runtime.error.ErrorHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -45,11 +43,10 @@ public class EventRuleTransfer extends ServiceThread {
private static final Logger logger = LoggerFactory.getLogger(EventRuleTransfer.class);
- @Autowired
+ private volatile Integer batchSize = 100;
+
private final CirculatorContext circulatorContext;
- @Autowired
private final OffsetManager offsetManager;
- @Autowired
private final ErrorHandler errorHandler;
public EventRuleTransfer(CirculatorContext circulatorContext, OffsetManager offsetManager,
@@ -72,9 +69,9 @@ public class EventRuleTransfer extends ServiceThread {
@Override
public void run() {
while (!stopped) {
- ConnectRecord eventRecord = circulatorContext.takeEventRecord();
- if (Objects.isNull(eventRecord)) {
- logger.info("listen eventRecord is empty, continue by curTime - {}", System.currentTimeMillis());
+ Map<String, List<ConnectRecord>> eventRecordMap = circulatorContext.takeEventRecords(batchSize);
+ if(MapUtils.isEmpty(eventRecordMap)){
+ logger.info("listen eventRecords is empty, continue by curTime - {}", System.currentTimeMillis());
this.waitForRunning(1000);
continue;
}
@@ -84,48 +81,40 @@ public class EventRuleTransfer extends ServiceThread {
this.waitForRunning(3000);
continue;
}
- // the event channel take rocket mq topic name as default
- String eventChannelName = RuntimerConfigDefine.CHANNEL_NAME;
- String eventChannel = eventRecord.getExtension(eventChannelName);
- Set<TransformEngine<ConnectRecord>> adaptTransformSet = latestTransformMap.values().stream()
- .filter(engine -> eventChannel.equals(engine.getConnectConfig(eventChannelName)))
- .collect(Collectors.toSet());
- if (CollectionUtils.isEmpty(adaptTransformSet)) {
- logger.warn("adapt specific topic ref transform engine is empty, eventChannelName- {}", eventChannel);
- this.waitForRunning(3000);
- continue;
- }
+
List<ConnectRecord> afterTransformConnect = Lists.newArrayList();
List<CompletableFuture<Void>> completableFutures = Lists.newArrayList();
- adaptTransformSet.forEach(transfer -> {
- CompletableFuture<Void> transformFuture = CompletableFuture.supplyAsync(() -> transfer.doTransforms(eventRecord))
- .exceptionally((exception) -> {
- logger.error("transfer do transform event record failed,stackTrace-", exception);
- errorHandler.handle(eventRecord,exception);
- return null;
- })
- .thenAccept(record -> {
- if (Objects.nonNull(record)) {
- String runnerNameKey = RuntimerConfigDefine.RUNNER_NAME;
- String taskClassKey = RuntimerConfigDefine.TASK_CLASS;
- record.getExtensions().put(runnerNameKey, transfer.getConnectConfig(runnerNameKey));
- record.getExtensions().put(taskClassKey, transfer.getConnectConfig(taskClassKey));
- afterTransformConnect.add(record);
- } else {
- offsetManager.commit(eventRecord);
- }
- });
- completableFutures.add(transformFuture);
- });
+ for(String runnerName: eventRecordMap.keySet()){
+ TransformEngine<ConnectRecord> curTransformEngine = latestTransformMap.get(runnerName);
+ List<ConnectRecord> curEventRecords = eventRecordMap.get(runnerName);
+ curEventRecords.forEach(pullRecord -> {
+ CompletableFuture<Void> transformFuture = CompletableFuture.supplyAsync(() -> curTransformEngine.doTransforms(pullRecord))
+ .exceptionally((exception) -> {
+ logger.error("transfer do transform event record failed,stackTrace-", exception);
+ errorHandler.handle(pullRecord, exception);
+ return null;
+ })
+ .thenAccept(pushRecord -> {
+ if (Objects.nonNull(pushRecord)) {
+ afterTransformConnect.add(pushRecord);
+ } else {
+ offsetManager.commit(pullRecord);
+ }
+ });
+ completableFutures.add(transformFuture);
+ });
+ }
try {
- CompletableFuture.allOf(completableFutures.toArray(new CompletableFuture[adaptTransformSet.size()])).get();
+ CompletableFuture.allOf(completableFutures.toArray(new CompletableFuture[eventRecordMap.values().size()])).get();
circulatorContext.offerTargetTaskQueue(afterTransformConnect);
logger.info("offer target task queues succeed, transforms - {}", JSON.toJSONString(afterTransformConnect));
} catch (Exception exception) {
logger.error("transfer event record failed, stackTrace-", exception);
+ afterTransformConnect.forEach(transferRecord -> errorHandler.handle(transferRecord, exception));
}
}
}
+
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventTargetPusher.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventTargetTrigger.java
similarity index 52%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventTargetPusher.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventTargetTrigger.java
index b3216da..a54ee39 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/EventTargetPusher.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/EventTargetTrigger.java
@@ -15,18 +15,21 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot;
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot;
import com.alibaba.fastjson.JSON;
-import com.google.common.collect.Lists;
import io.openmessaging.connector.api.component.task.sink.SinkTask;
import io.openmessaging.connector.api.data.ConnectRecord;
-import java.util.Objects;
+
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ExecutorService;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.CirculatorContext;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.ServiceThread;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.config.RuntimerConfigDefine;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.error.ErrorHandler;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.OffsetManager;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.CirculatorContext;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.ServiceThread;
+import org.apache.rocketmq.eventbridge.adapter.runtime.error.ErrorHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,16 +38,17 @@ import org.slf4j.LoggerFactory;
*
* @author artisan
*/
-public class EventTargetPusher extends ServiceThread {
+public class EventTargetTrigger extends ServiceThread {
- private static final Logger logger = LoggerFactory.getLogger(EventTargetPusher.class);
+ private static final Logger logger = LoggerFactory.getLogger(EventTargetTrigger.class);
private final CirculatorContext circulatorContext;
private final OffsetManager offsetManager;
private final ErrorHandler errorHandler;
+ private volatile Integer batchSize = 100;
- public EventTargetPusher(CirculatorContext circulatorContext, OffsetManager offsetManager,
- ErrorHandler errorHandler) {
+ public EventTargetTrigger(CirculatorContext circulatorContext, OffsetManager offsetManager,
+ ErrorHandler errorHandler) {
this.circulatorContext = circulatorContext;
this.offsetManager = offsetManager;
this.errorHandler = errorHandler;
@@ -53,33 +57,36 @@ public class EventTargetPusher extends ServiceThread {
@Override
public void run() {
while (!stopped) {
- ConnectRecord targetRecord = circulatorContext.takeTargetMap();
- if (Objects.isNull(targetRecord)) {
+ Map<String, List<ConnectRecord>> targetRecordMap = circulatorContext.takeTargetRecords(batchSize);
+ if (MapUtils.isEmpty(targetRecordMap)) {
logger.info("current target pusher is empty");
this.waitForRunning(1000);
continue;
}
if (logger.isDebugEnabled()) {
- logger.debug("start push content by pusher - {}", JSON.toJSONString(targetRecord));
+ logger.debug("start push content by pusher - {}", JSON.toJSONString(targetRecordMap));
}
- ExecutorService executorService = circulatorContext.getExecutorService(targetRecord.getExtensions().getString(RuntimerConfigDefine.TASK_CLASS));
- executorService.execute(() -> {
- try {
- String runnerName = targetRecord.getExtensions().getString(RuntimerConfigDefine.RUNNER_NAME);
+ for(String runnerName: targetRecordMap.keySet()){
+ ExecutorService executorService = circulatorContext.getExecutorService(runnerName);
+ executorService.execute(() -> {
SinkTask sinkTask = circulatorContext.getPusherTaskMap().get(runnerName);
- sinkTask.put(Lists.newArrayList(targetRecord));
- offsetManager.commit(targetRecord);
- } catch (Exception exception) {
- logger.error(getServiceName() + " push target exception, record - " + targetRecord + " , stackTrace-", exception);
- }
- });
+ List<ConnectRecord> triggerRecords = targetRecordMap.get(runnerName);
+ try {
+ sinkTask.put(triggerRecords);
+ offsetManager.commit(triggerRecords);
+ } catch (Exception exception) {
+ logger.error(getServiceName() + " push target exception, stackTrace-", exception);
+ triggerRecords.forEach(triggerRecord -> errorHandler.handle(triggerRecord, exception));
+ }
+ });
+ }
}
}
@Override
public String getServiceName() {
- return EventTargetPusher.class.getSimpleName();
+ return EventTargetTrigger.class.getSimpleName();
}
}
diff --git a/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/CirculatorContext.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/CirculatorContext.java
new file mode 100644
index 0000000..203f43a
--- /dev/null
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/CirculatorContext.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot.common;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import io.openmessaging.connector.api.component.task.sink.SinkTask;
+import io.openmessaging.connector.api.data.ConnectRecord;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.rocketmq.common.utils.ThreadUtils;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.trigger.TriggerTaskContext;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.transfer.TransformEngine;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.LoggerName;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetKeyValue;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.enums.RefreshTypeEnum;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.plugin.Plugin;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.plugin.PluginClassLoader;
+import org.apache.rocketmq.eventbridge.adapter.runtime.config.RuntimeConfigDefine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.*;
+
+/**
+ * event circulator context for listener, transfer and trigger
+ */
+@Component
+public class CirculatorContext implements TargetRunnerListener {
+
+ private final static Logger logger = LoggerFactory.getLogger(LoggerName.EventBus_Listener);
+
+ @Autowired
+ private Plugin plugin;
+
+ private static Integer QUEUE_CAPACITY = 50000;
+
+ private BlockingQueue<ConnectRecord> eventQueue = new LinkedBlockingQueue<>(50000);
+
+ private BlockingQueue<ConnectRecord> targetQueue = new LinkedBlockingQueue<>(50000);
+
+ private Map<String/*RunnerName*/, TargetRunnerConfig> runnerConfigMap = new ConcurrentHashMap<>(30);
+
+ private Map<String/*RunnerName*/, BlockingQueue<ConnectRecord>> eventQueueMap = new ConcurrentHashMap<>(30);
+
+ private Map<String/*RunnerName*/, BlockingQueue<ConnectRecord>> targetQueueMap = new ConcurrentHashMap<>(30);
+
+ private Map<String/*RunnerName*/, TransformEngine<ConnectRecord>> taskTransformMap = new ConcurrentHashMap<>(20);
+
+ private Map<String/*RunnerName*/, SinkTask> pusherTaskMap = new ConcurrentHashMap<>(20);
+
+ private Map<String/*RunnerName*/, ExecutorService> pusherExecutorMap = new ConcurrentHashMap<>(10);
+
+ /**
+ * initial targetRunnerMap, taskTransformMap, pusherTaskMap
+ * @param targetRunnerConfigs
+ */
+ public void initCirculatorContext(Set<TargetRunnerConfig> targetRunnerConfigs) {
+ if (CollectionUtils.isEmpty(targetRunnerConfigs)) {
+ return;
+ }
+ for (TargetRunnerConfig targetRunnerConfig : targetRunnerConfigs) {
+ onAddTargetRunner(targetRunnerConfig);
+ }
+ }
+
+ @Override
+ public void onAddTargetRunner(TargetRunnerConfig targetRunnerConfig) {
+ refreshRunnerContext(targetRunnerConfig, RefreshTypeEnum.ADD);
+ }
+
+ @Override
+ public void onUpdateTargetRunner(TargetRunnerConfig targetRunnerConfig) {
+ refreshRunnerContext(targetRunnerConfig, RefreshTypeEnum.UPDATE);
+ }
+
+ @Override
+ public void onDeleteTargetRunner(TargetRunnerConfig targetRunnerConfig) {
+ refreshRunnerContext(targetRunnerConfig, RefreshTypeEnum.DELETE);
+ }
+
+ /**
+ * get target runner config by runner name
+ * @param runnerName
+ * @return
+ */
+ public TargetRunnerConfig getRunnerConfig(String runnerName){
+ return runnerConfigMap.get(runnerName);
+ }
+
+ /**
+ * offer event records
+ *
+ * @param connectRecords
+ */
+ public boolean offerEventRecords(List<ConnectRecord> connectRecords) {
+ Map<String, List<ConnectRecord>> recordMap = buildWithRunnerNameKeyMap(connectRecords);
+ updateRecordQueueMap(recordMap, eventQueueMap);
+ return eventQueue.addAll(connectRecords);
+ }
+
+ /**
+ * update record queue map
+ * @param recordMap
+ * @param eventQueueMap
+ */
+ private boolean updateRecordQueueMap(Map<String, List<ConnectRecord>> recordMap, Map<String, BlockingQueue<ConnectRecord>> eventQueueMap) {
+ try{
+ for(String runnerName : recordMap.keySet()){
+ BlockingQueue<ConnectRecord> recordQueue = eventQueueMap.get(runnerName);
+ if(CollectionUtils.isEmpty(recordQueue)){
+ recordQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY);
+ }
+ recordQueue.addAll(recordMap.get(runnerName));
+ eventQueueMap.put(runnerName, recordQueue);
+ }
+ return true;
+ }catch (Exception exception){
+ return false;
+ }
+ }
+
+ /**
+ * take event records
+ *
+ * @return
+ */
+ public Map<String, List<ConnectRecord>> takeEventRecords(int batchSize) {
+ if(eventQueue.isEmpty()){
+ return null;
+ }
+ List<ConnectRecord> eventRecords = Lists.newArrayList();
+ eventQueue.drainTo(eventRecords, batchSize);
+ return buildWithRunnerNameKeyMap(eventRecords);
+ }
+
+ public Map<String, TransformEngine<ConnectRecord>> getTaskTransformMap() {
+ return taskTransformMap;
+ }
+
+ public Map<String, SinkTask> getPusherTaskMap() {
+ return pusherTaskMap;
+ }
+
+ public boolean offerTargetTaskQueue(List<ConnectRecord> connectRecords) {
+ Map<String, List<ConnectRecord>> recordMap = buildWithRunnerNameKeyMap(connectRecords);
+ updateRecordQueueMap(recordMap, targetQueueMap);
+ return targetQueue.addAll(connectRecords);
+ }
+
+ /**
+ * take batch target records
+ * @param batchSize
+ * @return
+ */
+ public Map<String, List<ConnectRecord>> takeTargetRecords(Integer batchSize) {
+ if(targetQueue.isEmpty()){
+ return null;
+ }
+ List<ConnectRecord> targetRecords = Lists.newArrayList();
+ targetQueue.drainTo(targetRecords, batchSize);
+ return buildWithRunnerNameKeyMap(targetRecords);
+ }
+
+ /**
+ * user runner-name as key
+ * @param eventRecords
+ * @return
+ */
+ private Map<String, List<ConnectRecord>> buildWithRunnerNameKeyMap(List<ConnectRecord> eventRecords) {
+ Map<String, List<ConnectRecord>> eventRecordMap = Maps.newHashMap();
+ for (ConnectRecord connectRecord : eventRecords) {
+ String runnerName = connectRecord.getExtension(RuntimeConfigDefine.RUNNER_NAME);
+ List<ConnectRecord> curEventRecords = eventRecordMap.get(runnerName);
+ if(CollectionUtils.isEmpty(curEventRecords)){
+ curEventRecords = Lists.newArrayList();
+ }
+ curEventRecords.add(connectRecord);
+ eventRecordMap.put(runnerName, curEventRecords);
+ }
+ return eventRecordMap;
+ }
+
+ /**
+ * get specific thread pool by push name
+ * @param runnerName
+ * @return
+ */
+ public ExecutorService getExecutorService(String runnerName){
+ return pusherExecutorMap.get(runnerName);
+ }
+
+ /**
+ * refresh target runner where config changed
+ * @param targetRunnerConfig
+ * @param refreshTypeEnum
+ */
+ private void refreshRunnerContext(TargetRunnerConfig targetRunnerConfig, RefreshTypeEnum refreshTypeEnum) {
+ String runnerName = targetRunnerConfig.getName();
+ switch (refreshTypeEnum) {
+ case ADD:
+ case UPDATE:
+ runnerConfigMap.put(runnerName, targetRunnerConfig);
+ TransformEngine<ConnectRecord> transformChain = new TransformEngine<>(targetRunnerConfig.getComponents(), plugin);
+ taskTransformMap.put(runnerName, transformChain);
+
+ int endIndex = targetRunnerConfig.getComponents().size() -1;
+ TargetKeyValue targetKeyValue = new TargetKeyValue(targetRunnerConfig.getComponents().get(endIndex));
+ SinkTask sinkTask = initTargetSinkTask(targetKeyValue);
+ pusherTaskMap.put(runnerName, sinkTask);
+
+ if (!pusherExecutorMap.containsKey(runnerName)) {
+ pusherExecutorMap.put(runnerName, initDefaultThreadPoolExecutor(runnerName));
+ }
+
+ if(logger.isInfoEnabled()){
+ logger.info("runnerName -{}- refresh context by refresh type -{}- succeed", runnerName, refreshTypeEnum.name());
+ }
+ break;
+ case DELETE:
+ runnerConfigMap.remove(runnerName);
+ taskTransformMap.remove(runnerName);
+ pusherTaskMap.remove(runnerName);
+ if(logger.isInfoEnabled()){
+ logger.info("runnerName -{}- remove context succeed", runnerName);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ /**
+ * init default thread poll param, support auto config
+ * @param threadPollName
+ * @return
+ */
+ private ExecutorService initDefaultThreadPoolExecutor(String threadPollName) {
+ return new ThreadPoolExecutor(200, 300, 1, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<>(300), ThreadUtils.newThreadFactory(threadPollName, false));
+ }
+
+ /**
+ * init target sink task
+ * @param targetKeyValue
+ * @return
+ */
+ private SinkTask initTargetSinkTask(TargetKeyValue targetKeyValue) {
+ String taskClass = targetKeyValue.getString(RuntimeConfigDefine.RUNNER_CLASS);
+ ClassLoader loader = plugin.getPluginClassLoader(taskClass);
+ Class taskClazz;
+ boolean isolationFlag = false;
+ try {
+ if (loader instanceof PluginClassLoader) {
+ taskClazz = ((PluginClassLoader) loader).loadClass(taskClass, false);
+ isolationFlag = true;
+ } else {
+ taskClazz = Class.forName(taskClass);
+ }
+ SinkTask sinkTask = (SinkTask) taskClazz.getDeclaredConstructor().newInstance();
+ sinkTask.init(targetKeyValue);
+ TriggerTaskContext sinkTaskContext = new TriggerTaskContext(targetKeyValue);
+ sinkTask.start(sinkTaskContext);
+ if (isolationFlag) {
+ Plugin.compareAndSwapLoaders(loader);
+ }
+ return sinkTask;
+ }catch (Exception exception) {
+ logger.error("task class -" + taskClass + "- init its sinkTask failed, ex- ", exception);
+ }
+ return null;
+ }
+
+}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/OffsetManager.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/OffsetManager.java
similarity index 81%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/OffsetManager.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/OffsetManager.java
index 0613efb..7abbfc6 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/OffsetManager.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/OffsetManager.java
@@ -15,24 +15,23 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot;
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot.common;
import com.google.common.collect.Lists;
import io.openmessaging.connector.api.data.ConnectRecord;
import java.util.List;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.EventSubscriber;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.listener.EventSubscriber;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+/**
+ * Manage the offset of event
+ */
@Component
public class OffsetManager {
@Autowired
- EventSubscriber eventSubscriber;
-
- public OffsetManager(EventSubscriber eventSubscriber) {
- this.eventSubscriber = eventSubscriber;
- }
+ private EventSubscriber eventSubscriber;
public void commit(final List<ConnectRecord> connectRecordList) {
this.eventSubscriber.commit(connectRecordList);
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/TargetRunnerListener.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/TargetRunnerListener.java
similarity index 89%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/TargetRunnerListener.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/TargetRunnerListener.java
index 8284e8a..ddc42c5 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/TargetRunnerListener.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/common/TargetRunnerListener.java
@@ -15,9 +15,9 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener;
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot.common;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
public interface TargetRunnerListener {
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/EventSubscriber.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/listener/EventSubscriber.java
similarity index 71%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/EventSubscriber.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/listener/EventSubscriber.java
index 0b33b9e..0d99fce 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/EventSubscriber.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/listener/EventSubscriber.java
@@ -15,22 +15,24 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener;
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot.listener;
import io.openmessaging.connector.api.data.ConnectRecord;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.enums.RefreshTypeEnum;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.TargetRunnerListener;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.SubscribeRunnerKeys;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.enums.RefreshTypeEnum;
import java.util.List;
public abstract class EventSubscriber implements TargetRunnerListener {
/**
- * Refresh subscriber inner data when runner config changed
- * @param targetRunnerConfig
+ * Refresh subscriber inner data when runner keys changed
+ * @param subscribeRunnerKeys
* @param refreshTypeEnum
*/
- public abstract void refresh(TargetRunnerConfig targetRunnerConfig, RefreshTypeEnum refreshTypeEnum);
+ public abstract void refresh(SubscribeRunnerKeys subscribeRunnerKeys, RefreshTypeEnum refreshTypeEnum);
/**
* Pull connect records from store, Blocking method when is empty.
@@ -64,7 +66,7 @@ public abstract class EventSubscriber implements TargetRunnerListener {
*/
@Override
public void onAddTargetRunner(TargetRunnerConfig targetRunnerConfig) {
- this.refresh(targetRunnerConfig, RefreshTypeEnum.ADD);
+ this.refresh(targetRunnerConfig.getSubscribeRunnerKeys(), RefreshTypeEnum.ADD);
}
/**
@@ -74,7 +76,7 @@ public abstract class EventSubscriber implements TargetRunnerListener {
*/
@Override
public void onUpdateTargetRunner(TargetRunnerConfig targetRunnerConfig) {
- this.refresh(targetRunnerConfig, RefreshTypeEnum.UPDATE);
+ this.refresh(targetRunnerConfig.getSubscribeRunnerKeys(), RefreshTypeEnum.UPDATE);
}
/**
@@ -84,7 +86,7 @@ public abstract class EventSubscriber implements TargetRunnerListener {
*/
@Override
public void onDeleteTargetRunner(TargetRunnerConfig targetRunnerConfig) {
- this.refresh(targetRunnerConfig, RefreshTypeEnum.DELETE);
+ this.refresh(targetRunnerConfig.getSubscribeRunnerKeys(), RefreshTypeEnum.DELETE);
}
}
\ No newline at end of file
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/transfer/TransformEngine.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/transfer/TransformEngine.java
similarity index 84%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/transfer/TransformEngine.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/transfer/TransformEngine.java
index 56f8549..f9e879a 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/transfer/TransformEngine.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/transfer/TransformEngine.java
@@ -15,18 +15,18 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot.transfer;
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot.transfer;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.component.Transform;
import io.openmessaging.connector.api.data.ConnectRecord;
import io.openmessaging.internal.DefaultKeyValue;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.LoggerName;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetKeyValue;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.plugin.Plugin;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.plugin.PluginClassLoader;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.config.RuntimerConfigDefine;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.LoggerName;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetKeyValue;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.plugin.Plugin;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.plugin.PluginClassLoader;
+import org.apache.rocketmq.eventbridge.adapter.runtime.config.RuntimeConfigDefine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,7 +37,7 @@ import java.util.Objects;
public class TransformEngine<R extends ConnectRecord> implements AutoCloseable {
- private static final Logger log = LoggerFactory.getLogger(LoggerName.EventBridge_RUNTIMER);
+ private static final Logger logger = LoggerFactory.getLogger(LoggerName.EventRule_Transfer);
private final List<Transform> transformList;
@@ -49,7 +49,7 @@ public class TransformEngine<R extends ConnectRecord> implements AutoCloseable {
private static final String COMMA = ",";
- private static final String PREFIX = RuntimerConfigDefine.TRANSFORMS + "-";
+ private static final String PREFIX = RuntimeConfigDefine.TRANSFORMS + "-";
public TransformEngine(List<Map<String, String>> transferConfigs, Plugin plugin) {
this.transferConfigs = transferConfigs;
@@ -63,12 +63,12 @@ public class TransformEngine<R extends ConnectRecord> implements AutoCloseable {
int endIndex = transferConfigs.size() - 1;
for (int index = 1; index < endIndex; index++) {
Map<String, String> transferMap = transferConfigs.get(index);
- String transformClass = transferMap.get(RuntimerConfigDefine.RUNNER_CLASS);
+ String transformClass = transferMap.get(RuntimeConfigDefine.RUNNER_CLASS);
try {
Transform transform = getTransform(transformClass);
KeyValue transformConfig = new DefaultKeyValue();
for (String key : transferMap.keySet()) {
- if (!key.equals(RuntimerConfigDefine.RUNNER_CLASS)) {
+ if (!key.equals(RuntimeConfigDefine.RUNNER_CLASS)) {
transformConfig.put(key, transferMap.get(key));
}
}
@@ -76,7 +76,7 @@ public class TransformEngine<R extends ConnectRecord> implements AutoCloseable {
transform.init(transformConfig);
this.transformList.add(transform);
} catch (Exception e) {
- log.error("transform new instance error", e);
+ logger.error("transform new instance error", e);
}
}
}
@@ -95,7 +95,7 @@ public class TransformEngine<R extends ConnectRecord> implements AutoCloseable {
// init listener key
TargetKeyValue targetKeyValue = new TargetKeyValue(components.get(startIndex));
// init pusher key
- targetKeyValue.put(RuntimerConfigDefine.TASK_CLASS, components.get(endIndex).get(RuntimerConfigDefine.RUNNER_CLASS));
+ targetKeyValue.put(RuntimeConfigDefine.TASK_CLASS, components.get(endIndex).get(RuntimeConfigDefine.RUNNER_CLASS));
return targetKeyValue;
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/pusher/PusherTaskContext.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/trigger/TriggerTaskContext.java
similarity index 74%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/pusher/PusherTaskContext.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/trigger/TriggerTaskContext.java
index dc17336..6af6382 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/pusher/PusherTaskContext.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/boot/trigger/TriggerTaskContext.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot.pusher;
+package org.apache.rocketmq.eventbridge.adapter.runtime.boot.trigger;
import com.alibaba.fastjson.JSON;
import io.openmessaging.connector.api.component.task.sink.SinkTaskContext;
@@ -25,9 +25,9 @@ import io.openmessaging.connector.api.data.RecordPartition;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetKeyValue;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.LoggerName;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.QueueState;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetKeyValue;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.LoggerName;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.QueueState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,14 +36,14 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-public class PusherTaskContext implements SinkTaskContext {
+public class TriggerTaskContext implements SinkTaskContext {
/**
* The configs of current sink task.
*/
private final TargetKeyValue taskConfig;
- private static final Logger log = LoggerFactory.getLogger(LoggerName.EventBridge_RUNTIMER);
+ private static final Logger logger = LoggerFactory.getLogger(LoggerName.EventTarget_Trigger);
private final Map<MessageQueue, Long> messageQueuesOffsetMap = new ConcurrentHashMap<>(64);
@@ -54,27 +54,27 @@ public class PusherTaskContext implements SinkTaskContext {
public static final String TOPIC = "topic";
public static final String QUEUE_OFFSET = "queueOffset";
- public PusherTaskContext(TargetKeyValue taskConfig) {
+ public TriggerTaskContext(TargetKeyValue taskConfig) {
this.taskConfig = taskConfig;
}
@Override
public void resetOffset(RecordPartition recordPartition, RecordOffset recordOffset) {
if (null == recordPartition || null == recordPartition.getPartition() || null == recordOffset || null == recordOffset.getOffset()) {
- log.warn("recordPartition {} info is null or recordOffset {} info is null", recordPartition, recordOffset);
+ logger.warn("recordPartition {} info is null or recordOffset {} info is null", recordPartition, recordOffset);
return;
}
String brokerName = (String) recordPartition.getPartition().get(BROKER_NAME);
String topic = (String) recordPartition.getPartition().get(TOPIC);
Integer queueId = Integer.valueOf((String) recordPartition.getPartition().get(QUEUE_ID));
if (StringUtils.isEmpty(brokerName) || StringUtils.isEmpty(topic) || null == queueId) {
- log.warn("brokerName is null or queueId is null or queueName is null, brokerName {}, queueId {} queueId {}", brokerName, queueId, topic);
+ logger.warn("brokerName is null or queueId is null or queueName is null, brokerName {}, queueId {} queueId {}", brokerName, queueId, topic);
return;
}
MessageQueue messageQueue = new MessageQueue(topic, brokerName, queueId);
Long offset = Long.valueOf((String) recordOffset.getOffset().get(QUEUE_OFFSET));
if (null == offset) {
- log.warn("resetOffset, offset is null");
+ logger.warn("resetOffset, offset is null");
return;
}
messageQueuesOffsetMap.put(messageQueue, offset);
@@ -83,12 +83,12 @@ public class PusherTaskContext implements SinkTaskContext {
@Override
public void resetOffset(Map<RecordPartition, RecordOffset> offsets) {
if (MapUtils.isEmpty(offsets)) {
- log.warn("resetOffset, offsets {} is null", offsets);
+ logger.warn("resetOffset, offsets {} is null", offsets);
return;
}
for (Map.Entry<RecordPartition, RecordOffset> entry : offsets.entrySet()) {
if (null == entry || null == entry.getKey() || null == entry.getKey().getPartition() || null == entry.getValue() || null == entry.getValue().getOffset()) {
- log.warn("recordPartition {} info is null or recordOffset {} info is null, entry {}", entry);
+ logger.warn("recordPartition {} info is null or recordOffset {} info is null, entry {}", entry);
continue;
}
RecordPartition recordPartition = entry.getKey();
@@ -96,14 +96,14 @@ public class PusherTaskContext implements SinkTaskContext {
String topic = (String) recordPartition.getPartition().get(TOPIC);
Integer queueId = Integer.valueOf((String) recordPartition.getPartition().get(QUEUE_ID));
if (StringUtils.isEmpty(brokerName) || StringUtils.isEmpty(topic) || null == queueId) {
- log.warn("brokerName is null or queueId is null or queueName is null, brokerName {}, queueId {} queueId {}", brokerName, queueId, topic);
+ logger.warn("brokerName is null or queueId is null or queueName is null, brokerName {}, queueId {} queueId {}", brokerName, queueId, topic);
continue;
}
MessageQueue messageQueue = new MessageQueue(topic, brokerName, queueId);
RecordOffset recordOffset = entry.getValue();
Long offset = Long.valueOf((String) recordOffset.getOffset().get(QUEUE_OFFSET));
if (null == offset) {
- log.warn("resetOffset, offset is null");
+ logger.warn("resetOffset, offset is null");
continue;
}
messageQueuesOffsetMap.put(messageQueue, offset);
@@ -113,24 +113,24 @@ public class PusherTaskContext implements SinkTaskContext {
@Override
public void pause(List<RecordPartition> recordPartitions) {
if (recordPartitions == null || recordPartitions.size() == 0) {
- log.warn("recordPartitions is null or recordPartitions.size() is zero. recordPartitions {}", JSON.toJSONString(recordPartitions));
+ logger.warn("recordPartitions is null or recordPartitions.size() is zero. recordPartitions {}", JSON.toJSONString(recordPartitions));
return;
}
for (RecordPartition recordPartition : recordPartitions) {
if (null == recordPartition || null == recordPartition.getPartition()) {
- log.warn("recordPartition {} info is null", recordPartition);
+ logger.warn("recordPartition {} info is null", recordPartition);
continue;
}
String brokerName = (String) recordPartition.getPartition().get(BROKER_NAME);
String topic = (String) recordPartition.getPartition().get(TOPIC);
Integer queueId = Integer.valueOf((String) recordPartition.getPartition().get(QUEUE_ID));
if (StringUtils.isEmpty(brokerName) || StringUtils.isEmpty(topic) || null == queueId) {
- log.warn("brokerName is null or queueId is null or queueName is null, brokerName {}, queueId {} queueId {}", brokerName, queueId, topic);
+ logger.warn("brokerName is null or queueId is null or queueName is null, brokerName {}, queueId {} queueId {}", brokerName, queueId, topic);
continue;
}
MessageQueue messageQueue = new MessageQueue(topic, brokerName, queueId);
if (!messageQueuesOffsetMap.containsKey(messageQueue)) {
- log.warn("sink task current messageQueuesOffsetMap {} not contain messageQueue {}", messageQueuesOffsetMap, messageQueue);
+ logger.warn("sink task current messageQueuesOffsetMap {} not contain messageQueue {}", messageQueuesOffsetMap, messageQueue);
continue;
}
messageQueuesStateMap.put(messageQueue, QueueState.PAUSE);
@@ -140,24 +140,24 @@ public class PusherTaskContext implements SinkTaskContext {
@Override
public void resume(List<RecordPartition> recordPartitions) {
if (recordPartitions == null || recordPartitions.size() == 0) {
- log.warn("recordPartitions is null or recordPartitions.size() is zero. recordPartitions {}", JSON.toJSONString(recordPartitions));
+ logger.warn("recordPartitions is null or recordPartitions.size() is zero. recordPartitions {}", JSON.toJSONString(recordPartitions));
return;
}
for (RecordPartition recordPartition : recordPartitions) {
if (null == recordPartition || null == recordPartition.getPartition()) {
- log.warn("recordPartition {} info is null", recordPartition);
+ logger.warn("recordPartition {} info is null", recordPartition);
continue;
}
String brokerName = (String) recordPartition.getPartition().get(BROKER_NAME);
String topic = (String) recordPartition.getPartition().get(TOPIC);
Integer queueId = Integer.valueOf((String) recordPartition.getPartition().get(QUEUE_ID));
if (StringUtils.isEmpty(brokerName) || StringUtils.isEmpty(topic) || null == queueId) {
- log.warn("brokerName is null or queueId is null or queueName is null, brokerName {}, queueId {} queueId {}", brokerName, queueId, topic);
+ logger.warn("brokerName is null or queueId is null or queueName is null, brokerName {}, queueId {} queueId {}", brokerName, queueId, topic);
continue;
}
MessageQueue messageQueue = new MessageQueue(topic, brokerName, queueId);
if (!messageQueuesOffsetMap.containsKey(messageQueue)) {
- log.warn("sink task current messageQueuesOffsetMap {} not contain messageQueue {}", messageQueuesOffsetMap, messageQueue);
+ logger.warn("sink task current messageQueuesOffsetMap {} not contain messageQueue {}", messageQueuesOffsetMap, messageQueue);
continue;
}
messageQueuesStateMap.remove(messageQueue);
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/FileAndPropertyUtil.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/FileAndPropertyUtil.java
similarity index 98%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/FileAndPropertyUtil.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/FileAndPropertyUtil.java
index 6b121d7..71b7e95 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/FileAndPropertyUtil.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/FileAndPropertyUtil.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common;
import java.io.File;
import java.io.FileInputStream;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/FilePathConfigUtil.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/FilePathConfigUtil.java
similarity index 95%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/FilePathConfigUtil.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/FilePathConfigUtil.java
index 9156a99..09499f1 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/FilePathConfigUtil.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/FilePathConfigUtil.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common;
import java.io.File;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/LoggerName.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/LoggerName.java
similarity index 88%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/LoggerName.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/LoggerName.java
index aa1b5d9..39734c4 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/LoggerName.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/LoggerName.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common;
/**
* Define all the logger name of the runtime.
@@ -24,5 +24,5 @@ public class LoggerName {
public static final String EventBridge_RUNTIMER = "EventBridgeRuntimer";
public static final String EventBus_Listener = "EventBusListener";
public static final String EventRule_Transfer = "EventRuleTransfer";
- public static final String EventTarget_Pusher = "EventTargetPusher";
+ public static final String EventTarget_Trigger = "EventTargetTrigger";
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/QueueState.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/QueueState.java
similarity index 92%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/QueueState.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/QueueState.java
index 9c00e53..8665b5e 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/QueueState.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/QueueState.java
@@ -16,7 +16,7 @@
*
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common;
public enum QueueState {
PAUSE;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/RuntimerState.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/RuntimeState.java
similarity index 88%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/RuntimerState.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/RuntimeState.java
index daffff1..5a06c92 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/RuntimerState.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/RuntimeState.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common;
/**
- * 运行器状态
+ * Runtime state
*/
-public enum RuntimerState {
+public enum RuntimeState {
START,
STOP,
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/ServiceThread.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/ServiceThread.java
similarity index 97%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/ServiceThread.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/ServiceThread.java
index 566d8e2..48835c1 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/ServiceThread.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/ServiceThread.java
@@ -15,9 +15,8 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common;
-import javax.annotation.PostConstruct;
import org.apache.rocketmq.common.CountDownLatch2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/SubscribeRunnerKeys.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/SubscribeRunnerKeys.java
new file mode 100644
index 0000000..d708c36
--- /dev/null
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/SubscribeRunnerKeys.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.entity;
+
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ * Support target runner config with few keys for subscriber
+ */
+@Data
+public class SubscribeRunnerKeys implements Serializable {
+
+ private String accountId;
+
+ private String runnerName;
+
+ private String eventBusName;
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ SubscribeRunnerKeys that = (SubscribeRunnerKeys) o;
+ return Objects.equals(accountId, that.accountId) && Objects.equals(runnerName, that.runnerName) && Objects.equals(eventBusName, that.eventBusName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(accountId, runnerName, eventBusName);
+ }
+
+ @Override
+ public String toString() {
+ return "SubscribeRunnerKeys{" +
+ "accountId='" + accountId + '\'' +
+ ", runnerName='" + runnerName + '\'' +
+ ", eventBusName='" + eventBusName + '\'' +
+ '}';
+ }
+}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/entity/TargetKeyValue.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/TargetKeyValue.java
similarity index 98%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/entity/TargetKeyValue.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/TargetKeyValue.java
index 956d48e..aa696e8 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/entity/TargetKeyValue.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/TargetKeyValue.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.entity;
import io.openmessaging.KeyValue;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/entity/TargetRunnerConfig.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/TargetRunnerConfig.java
similarity index 83%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/entity/TargetRunnerConfig.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/TargetRunnerConfig.java
index d97a147..6cbc2f7 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/entity/TargetRunnerConfig.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/entity/TargetRunnerConfig.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.entity;
import java.io.Serializable;
import java.util.List;
@@ -27,7 +27,8 @@ import org.apache.rocketmq.eventbridge.domain.model.run.RunOptions;
import org.apache.rocketmq.eventbridge.enums.ErrorToleranceEnum;
import org.apache.rocketmq.eventbridge.enums.PushRetryStrategyEnum;
-import static org.apache.rocketmq.eventbridge.adapter.runtimer.config.RuntimerConfigDefine.TARGET_RUNNER_KEY;
+import static org.apache.rocketmq.eventbridge.adapter.runtime.config.RuntimeConfigDefine.TARGET_RUNNER_KEY;
+import static org.apache.rocketmq.eventbridge.adapter.runtime.config.RuntimeConfigDefine.ACCOUNT_ID;
/**
* pusher target key config
@@ -111,4 +112,16 @@ public class TargetRunnerConfig implements Serializable {
public String getEventBusName() {
return components.get(0).get(TARGET_RUNNER_KEY);
}
+
+ public String getAccountId() {
+ return components.get(0).get(ACCOUNT_ID);
+ }
+
+ public SubscribeRunnerKeys getSubscribeRunnerKeys(){
+ SubscribeRunnerKeys subscribeRunnerKeys = new SubscribeRunnerKeys();
+ subscribeRunnerKeys.setRunnerName(this.getName());
+ subscribeRunnerKeys.setAccountId(this.getAccountId());
+ subscribeRunnerKeys.setEventBusName(this.getEventBusName());
+ return subscribeRunnerKeys;
+ }
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/ConfigModeEnum.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/ConfigModeEnum.java
similarity index 94%
copy from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/ConfigModeEnum.java
copy to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/ConfigModeEnum.java
index f925ed3..d50a92d 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/ConfigModeEnum.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/ConfigModeEnum.java
@@ -16,7 +16,7 @@
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.enums;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.enums;
import com.google.common.base.Strings;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/RefreshTypeEnum.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/RefreshTypeEnum.java
similarity index 92%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/RefreshTypeEnum.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/RefreshTypeEnum.java
index 9e8f6cd..fa9ee42 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/RefreshTypeEnum.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/RefreshTypeEnum.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.enums;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.enums;
/**
* runner refresh type
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/ConfigModeEnum.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/StorageModeEnum.java
similarity index 74%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/ConfigModeEnum.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/StorageModeEnum.java
index f925ed3..7b18543 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/enums/ConfigModeEnum.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/enums/StorageModeEnum.java
@@ -15,25 +15,27 @@
* limitations under the License.
*/
-
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.enums;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.enums;
import com.google.common.base.Strings;
-public enum ConfigModeEnum {
- FILE,
- DB,
- CONTROLLER;
+/**
+ * storage mode for event
+ */
+public enum StorageModeEnum {
+ ROCKETMQ,
+ KAFKA,
+ OTHER;
- public static ConfigModeEnum parse(String code) {
+ public static StorageModeEnum parse(String code) {
if (Strings.isNullOrEmpty(code)) {
- return FILE;
+ return ROCKETMQ;
}
- for (ConfigModeEnum configModeEnum : ConfigModeEnum.values()) {
+ for (StorageModeEnum configModeEnum : StorageModeEnum.values()) {
if (configModeEnum.name().equals(code)) {
return configModeEnum;
}
}
- return FILE;
+ return ROCKETMQ;
}
}
\ No newline at end of file
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/Plugin.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/Plugin.java
similarity index 98%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/Plugin.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/Plugin.java
index 89a3b99..26475f2 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/Plugin.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/Plugin.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.plugin;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.plugin;
import io.openmessaging.connector.api.component.Transform;
import io.openmessaging.connector.api.component.connector.Connector;
@@ -45,7 +45,7 @@ import java.util.*;
public class Plugin extends URLClassLoader {
private static final Logger log = LoggerFactory.getLogger(Plugin.class);
- @Value("${runtimer.pluginpath:}")
+ @Value("${runtime.pluginpath:}")
private String pluginPath;
private Map<String, PluginWrapper> classLoaderMap = new HashMap<>();
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginClassLoader.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginClassLoader.java
similarity index 97%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginClassLoader.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginClassLoader.java
index 428351e..fd0f699 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginClassLoader.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginClassLoader.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.plugin;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.plugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginUtils.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginUtils.java
similarity index 99%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginUtils.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginUtils.java
index effc821..2c11e1a 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginUtils.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginUtils.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.plugin;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.plugin;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginWrapper.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginWrapper.java
similarity index 97%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginWrapper.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginWrapper.java
index 2acaef6..2f81c65 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/plugin/PluginWrapper.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/plugin/PluginWrapper.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.plugin;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.plugin;
import io.openmessaging.connector.api.component.task.sink.SinkConnector;
import io.openmessaging.connector.api.component.task.source.SourceConnector;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/FileBaseKeyValueStore.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/FileBaseKeyValueStore.java
similarity index 94%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/FileBaseKeyValueStore.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/FileBaseKeyValueStore.java
index 61654c8..e16d4ec 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/FileBaseKeyValueStore.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/FileBaseKeyValueStore.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.store;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.store;
import com.alibaba.fastjson.JSON;
import io.openmessaging.connector.api.data.Converter;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.FileAndPropertyUtil;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.LoggerName;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.FileAndPropertyUtil;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.LoggerName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/KeyValueStore.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/KeyValueStore.java
similarity index 96%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/KeyValueStore.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/KeyValueStore.java
index d136589..6495db6 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/KeyValueStore.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/KeyValueStore.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.store;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.store;
import java.util.Map;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/MemoryBasedKeyValueStore.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/MemoryBasedKeyValueStore.java
similarity index 92%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/MemoryBasedKeyValueStore.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/MemoryBasedKeyValueStore.java
index 06c5258..b85f089 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/common/store/MemoryBasedKeyValueStore.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/common/store/MemoryBasedKeyValueStore.java
@@ -15,13 +15,13 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.common.store;
+package org.apache.rocketmq.eventbridge.adapter.runtime.common.store;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-public class MemoryBasedKeyValueStore<K, V> implements org.apache.rocketmq.eventbridge.adapter.runtimer.common.store.KeyValueStore<K, V> {
+public class MemoryBasedKeyValueStore<K, V> implements org.apache.rocketmq.eventbridge.adapter.runtime.common.store.KeyValueStore<K, V> {
protected Map<K, V> data;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceStrategy.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfigDefine.java
similarity index 53%
copy from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceStrategy.java
copy to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfigDefine.java
index 4020ac3..847f830 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceStrategy.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfigDefine.java
@@ -13,21 +13,32 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
+ *
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.trace;
-
+package org.apache.rocketmq.eventbridge.adapter.runtime.config;
/**
- * EventMeshTraceService
- * SPI可扩展
- * 基于OpenTelemetry实现封装不同追踪器
+ * Define keys for target runner configs.
*/
-public interface TraceStrategy {
+public class RuntimeConfigDefine {
+
+ public static final String RUNNER_CLASS = "class";
+
+ public static final String TASK_CLASS = "task-class";
+
+ public static final String RUNNER_NAME = "runner-name";
+
+ public static final String CONNECT_TIMESTAMP = "connect-timestamp";
+
+ public static final String CONNECT_SCHEMA = "connect-schema";
+
+ public static final String TRANSFORMS = "transforms";
+
+ public static final String CONNECT_RECORDS_KEY = "SYSTEM_RETRY_TIMES";
- void init() throws TraceException;
+ public static final String TARGET_RUNNER_KEY = "eventBusName";
- void createSpan(String spanName) throws TraceException;
+ public static final String ACCOUNT_ID = "accountId";
- void shutdown() throws TraceException;
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfigProps.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfigProps.java
similarity index 81%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfigProps.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfigProps.java
index 4f7827c..ea5fd11 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfigProps.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfigProps.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.config;
+package org.apache.rocketmq.eventbridge.adapter.runtime.config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -27,25 +27,25 @@ import java.util.Properties;
/**
* runtimer properties factory
*/
-public class RuntimerConfigProps {
+public class RuntimeConfigProps {
- private final static Logger logger = LoggerFactory.getLogger(RuntimerConfigProps.class);
+ private final static Logger logger = LoggerFactory.getLogger(RuntimeConfigProps.class);
private Properties properties;
- private RuntimerConfigProps(){
+ private RuntimeConfigProps(){
try {
- properties = PropertiesLoaderUtils.loadAllProperties("runtimer.properties");
+ properties = PropertiesLoaderUtils.loadAllProperties("runtime.properties");
} catch (IOException exception) {
logger.error("runtime load properties failed, stackTrace-", exception);
}
}
private static class RuntimerConfigPropsHolder{
- private static final RuntimerConfigProps instance = new RuntimerConfigProps();
+ private static final RuntimeConfigProps instance = new RuntimeConfigProps();
}
- public static RuntimerConfigProps build(){
+ public static RuntimeConfigProps build(){
return RuntimerConfigPropsHolder.instance;
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfiguration.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfiguration.java
similarity index 71%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfiguration.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfiguration.java
index 7160cca..b6a7a46 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfiguration.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/RuntimeConfiguration.java
@@ -15,21 +15,21 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.config;
+package org.apache.rocketmq.eventbridge.adapter.runtime.config;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.enums.ConfigModeEnum;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.service.TargetRunnerConfigObserver;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.service.TargetRunnerConfigOnDBObserver;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.service.TargetRunnerConfigOnFileObserver;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.enums.ConfigModeEnum;
+import org.apache.rocketmq.eventbridge.adapter.runtime.service.TargetRunnerConfigObserver;
+import org.apache.rocketmq.eventbridge.adapter.runtime.service.TargetRunnerConfigOnDBObserver;
+import org.apache.rocketmq.eventbridge.adapter.runtime.service.TargetRunnerConfigOnFileObserver;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
-public class RuntimerConfiguration {
+public class RuntimeConfiguration {
@Bean(name = "runnerConfigObserver")
- public TargetRunnerConfigObserver targetRunnerConfigObserver(@Value("${rumtimer.config.mode}") String configMode) {
+ public TargetRunnerConfigObserver targetRunnerConfigObserver(@Value("${runtime.config.mode}") String configMode) {
switch (ConfigModeEnum.parse(configMode)) {
case DB:
return new TargetRunnerConfigOnDBObserver();
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/TraceConfig.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/TraceConfig.java
similarity index 92%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/TraceConfig.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/TraceConfig.java
index d8b0dbb..9b8e9bb 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/TraceConfig.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/config/TraceConfig.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.config;
+package org.apache.rocketmq.eventbridge.adapter.runtime.config;
public class TraceConfig {
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/JsonConverter.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/JsonConverter.java
similarity index 94%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/JsonConverter.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/JsonConverter.java
index efc7d39..dfad654 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/JsonConverter.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/JsonConverter.java
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.converter;
+package org.apache.rocketmq.eventbridge.adapter.runtime.converter;
import com.alibaba.fastjson.JSON;
import io.openmessaging.connector.api.data.Converter;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.LoggerName;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.LoggerName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/ListConverter.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/ListConverter.java
similarity index 93%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/ListConverter.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/ListConverter.java
index 3ee9929..b704c7b 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/ListConverter.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/ListConverter.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.converter;
+package org.apache.rocketmq.eventbridge.adapter.runtime.converter;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import io.openmessaging.connector.api.data.Converter;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.LoggerName;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.LoggerName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/RecordOffsetConverter.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/RecordOffsetConverter.java
similarity index 93%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/RecordOffsetConverter.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/RecordOffsetConverter.java
index f039d9c..4a6d9ef 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/converter/RecordOffsetConverter.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/converter/RecordOffsetConverter.java
@@ -15,12 +15,12 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.converter;
+package org.apache.rocketmq.eventbridge.adapter.runtime.converter;
import com.alibaba.fastjson.JSON;
import io.openmessaging.connector.api.data.Converter;
import io.openmessaging.connector.api.data.RecordOffset;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.LoggerName;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.LoggerName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/error/DeadLetterQueueService.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/error/DeadLetterQueueService.java
similarity index 92%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/error/DeadLetterQueueService.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/error/DeadLetterQueueService.java
index fd108c9..3b10700 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/error/DeadLetterQueueService.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/error/DeadLetterQueueService.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.error;
+package org.apache.rocketmq.eventbridge.adapter.runtime.error;
public class DeadLetterQueueService {
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/error/ErrorHandler.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/error/ErrorHandler.java
similarity index 78%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/error/ErrorHandler.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/error/ErrorHandler.java
index 8a59c35..b7cee55 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/error/ErrorHandler.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/error/ErrorHandler.java
@@ -15,35 +15,38 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.error;
+package org.apache.rocketmq.eventbridge.adapter.runtime.error;
import com.google.common.base.Strings;
import io.openmessaging.connector.api.data.ConnectRecord;
import lombok.extern.slf4j.Slf4j;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.OffsetManager;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.EventSubscriber;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.TargetRunnerContext;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.CirculatorContext;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.OffsetManager;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.listener.EventSubscriber;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
import org.apache.rocketmq.eventbridge.enums.PushRetryStrategyEnum;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import static org.apache.rocketmq.eventbridge.adapter.runtimer.config.RuntimerConfigDefine.CONNECT_RECORDS_KEY;
-import static org.apache.rocketmq.eventbridge.adapter.runtimer.config.RuntimerConfigDefine.RUNNER_NAME;
+import static org.apache.rocketmq.eventbridge.adapter.runtime.config.RuntimeConfigDefine.CONNECT_RECORDS_KEY;
+import static org.apache.rocketmq.eventbridge.adapter.runtime.config.RuntimeConfigDefine.RUNNER_NAME;
@Slf4j
@Component
public class ErrorHandler {
@Autowired
- EventSubscriber eventSubscriber;
+ private EventSubscriber eventSubscriber;
@Autowired
- OffsetManager offsetManager;
+ private CirculatorContext circulatorContext;
+
+ @Autowired
+ private OffsetManager offsetManager;
public void handle(ConnectRecord connectRecord, Throwable t) {
String eventRunnerName = connectRecord.getExtension(RUNNER_NAME);
- TargetRunnerConfig targetRunnerConfig = TargetRunnerContext.getTargetRunnerConfig(eventRunnerName);
+ TargetRunnerConfig targetRunnerConfig = circulatorContext.getRunnerConfig(eventRunnerName);
String eventBusName = targetRunnerConfig.getEventBusName();
PushRetryStrategyEnum pushRetryStrategyEnum = targetRunnerConfig.getRunOptions().getRetryStrategy().getPushRetryStrategy();
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/AbstractTargetRunnerConfigObserver.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/AbstractTargetRunnerConfigObserver.java
similarity index 81%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/AbstractTargetRunnerConfigObserver.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/AbstractTargetRunnerConfigObserver.java
index 7157708..0615e35 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/AbstractTargetRunnerConfigObserver.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/AbstractTargetRunnerConfigObserver.java
@@ -15,15 +15,18 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.service;
+package org.apache.rocketmq.eventbridge.adapter.runtime.service;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.TargetRunnerListener;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
+import java.util.stream.Collectors;
+
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.TargetRunnerListener;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.SubscribeRunnerKeys;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
@@ -41,6 +44,20 @@ public abstract class AbstractTargetRunnerConfigObserver implements TargetRunner
return targetRunnerConfigs;
}
+ @Override
+ public Set<SubscribeRunnerKeys> getSubscribeRunnerKeys() {
+ if(CollectionUtils.isEmpty(targetRunnerConfigs)){
+ return null;
+ }
+ return targetRunnerConfigs.stream().map(item -> {
+ SubscribeRunnerKeys subscribeRunnerKeys = new SubscribeRunnerKeys();
+ subscribeRunnerKeys.setRunnerName(item.getName());
+ subscribeRunnerKeys.setAccountId(item.getAccountId());
+ subscribeRunnerKeys.setEventBusName(item.getEventBusName());
+ return subscribeRunnerKeys;
+ }).collect(Collectors.toSet());
+ }
+
public abstract Set<TargetRunnerConfig> getLatestTargetRunnerConfig();
@Override
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigObserver.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigObserver.java
similarity index 71%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigObserver.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigObserver.java
index 6c713e3..aaa0558 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigObserver.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigObserver.java
@@ -15,14 +15,13 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.service;
+package org.apache.rocketmq.eventbridge.adapter.runtime.service;
+
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.TargetRunnerListener;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.SubscribeRunnerKeys;
-import java.util.List;
-import java.util.Map;
import java.util.Set;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.TargetRunnerListener;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetKeyValue;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
/**
* manage the pusher connector/task config info
@@ -35,6 +34,12 @@ public interface TargetRunnerConfigObserver {
*/
Set<TargetRunnerConfig> getTargetRunnerConfig();
+ /**
+ * Get the target runner key which relevant as event bus name
+ * @return
+ */
+ Set<SubscribeRunnerKeys> getSubscribeRunnerKeys();
+
/**
* Register a listener to listen all config update operations.
*
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnControllerObserver.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnControllerObserver.java
similarity index 83%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnControllerObserver.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnControllerObserver.java
index f51ac50..e106033 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnControllerObserver.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnControllerObserver.java
@@ -15,15 +15,13 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.service;
+package org.apache.rocketmq.eventbridge.adapter.runtime.service;
import com.google.common.collect.Sets;
-import java.util.List;
-import java.util.Map;
+
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetKeyValue;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
@Slf4j
public class TargetRunnerConfigOnControllerObserver extends AbstractTargetRunnerConfigObserver {
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnDBObserver.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnDBObserver.java
similarity index 94%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnDBObserver.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnDBObserver.java
index 071ca48..c272bd3 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnDBObserver.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnDBObserver.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.service;
+package org.apache.rocketmq.eventbridge.adapter.runtime.service;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
@@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.apache.rocketmq.common.utils.ThreadUtils;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
import org.apache.rocketmq.eventbridge.domain.model.run.EventTargetRunner;
import org.apache.rocketmq.eventbridge.domain.repository.EventTargetRepository;
import org.apache.rocketmq.eventbridge.domain.repository.EventTargetRunnerRepository;
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnFileObserver.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnFileObserver.java
similarity index 93%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnFileObserver.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnFileObserver.java
index 3948ff7..b5d34f3 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/service/TargetRunnerConfigOnFileObserver.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/service/TargetRunnerConfigOnFileObserver.java
@@ -15,9 +15,8 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.service;
+package org.apache.rocketmq.eventbridge.adapter.runtime.service;
-import com.google.common.collect.Maps;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import java.io.File;
@@ -30,19 +29,16 @@ import java.nio.file.StandardWatchEventKinds;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.common.utils.ThreadUtils;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
import org.apache.rocketmq.eventbridge.exception.EventBridgeException;
import org.springframework.stereotype.Component;
-import org.springframework.stereotype.Service;
@Slf4j
@Component
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/JaegerTraceStrategy.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/JaegerTraceStrategy.java
similarity index 92%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/JaegerTraceStrategy.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/JaegerTraceStrategy.java
index e29c09b..8897b38 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/JaegerTraceStrategy.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/JaegerTraceStrategy.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.trace;
+package org.apache.rocketmq.eventbridge.adapter.runtime.trace;
public class JaegerTraceStrategy {
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceConstants.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/TraceConstants.java
similarity index 92%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceConstants.java
rename to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/TraceConstants.java
index 7adeafe..09b738c 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceConstants.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/TraceConstants.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.trace;
+package org.apache.rocketmq.eventbridge.adapter.runtime.trace;
public class TraceConstants {
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceFactory.java b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/TraceFactory.java
similarity index 93%
copy from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceFactory.java
copy to adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/TraceFactory.java
index f4f3b7d..db22449 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceFactory.java
+++ b/adapter/runtime/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtime/trace/TraceFactory.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.trace;
+package org.apache.rocketmq.eventbridge.adapter.runtime.trace;
import java.util.Map;
diff --git a/adapter/runtimer/src/main/resources/runtimer.properties b/adapter/runtime/src/main/resources/runtime.properties
similarity index 95%
rename from adapter/runtimer/src/main/resources/runtimer.properties
rename to adapter/runtime/src/main/resources/runtime.properties
index 7b290e5..b99c83d 100644
--- a/adapter/runtimer/src/main/resources/runtimer.properties
+++ b/adapter/runtime/src/main/resources/runtime.properties
@@ -18,7 +18,7 @@ rocketmq.namesrvAddr=localhost:9876
rocketmq.consumer.pullTimeOut = 3000
rocketmq.consumer.pullBatchSize=20
rocketmq.cluster.name=DefaultCluster
-## runtimer
+## runtime
rumtimer.name=eventbridge-runtimer
runtimer.pluginpath=/Users/Local/eventbridge/plugin
runtimer.storePathRootDir=/Users/Local/eventbridge/store
diff --git a/adapter/runtimer/src/main/resources/target-runner.json b/adapter/runtime/src/main/resources/target-runner.json
similarity index 88%
rename from adapter/runtimer/src/main/resources/target-runner.json
rename to adapter/runtime/src/main/resources/target-runner.json
index 6ffcaf7..d5ccbc0 100644
--- a/adapter/runtimer/src/main/resources/target-runner.json
+++ b/adapter/runtime/src/main/resources/target-runner.json
@@ -3,8 +3,8 @@
"name":"demo-runner",
"components":[
{
- "runner-name": "demo-runner",
- "channel-name":"eventbridge%654321%demo-bus_1678348282165"
+ "accountId": "654321",
+ "eventBusName":"demo-bus"
},
{
"filterPattern":"{}",
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigOnFileObserverTest.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigOnFileObserverTest.java
similarity index 94%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigOnFileObserverTest.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigOnFileObserverTest.java
index 41f1989..35e5cf6 100644
--- a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigOnFileObserverTest.java
+++ b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigOnFileObserverTest.java
@@ -18,9 +18,9 @@
package org.apache.rocketmq.eventbridge.runtimer.service;
import java.time.Duration;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.TargetRunnerListener;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.service.TargetRunnerConfigOnFileObserver;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.common.TargetRunnerListener;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.service.TargetRunnerConfigOnFileObserver;
import org.junit.Assert;
import org.junit.Test;
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigUtil.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigUtil.java
similarity index 98%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigUtil.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigUtil.java
index a837d52..d12b07f 100644
--- a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigUtil.java
+++ b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TargetRunnerConfigUtil.java
@@ -33,7 +33,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
import org.apache.commons.io.FileUtils;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.TargetRunnerConfig;
import org.apache.rocketmq.eventbridge.exception.EventBridgeException;
import static org.apache.rocketmq.eventbridge.runtimer.service.TestConstants.DEMO_TARGET_RUNNER_NAME;
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TestConstants.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TestConstants.java
similarity index 100%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TestConstants.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/TestConstants.java
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/ConsumerSlowerPIDControllerTest.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/ConsumerSlowerPIDControllerTest.java
similarity index 100%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/ConsumerSlowerPIDControllerTest.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/ConsumerSlowerPIDControllerTest.java
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/MockEventBusListenerTest.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/MockEventBusListenerTest.java
similarity index 96%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/MockEventBusListenerTest.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/MockEventBusListenerTest.java
index a1910d3..ba7e6e5 100644
--- a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/MockEventBusListenerTest.java
+++ b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/MockEventBusListenerTest.java
@@ -16,7 +16,7 @@
*/
package org.apache.rocketmq.eventbridge.runtimer.service.backpressure;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.ServiceThread;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.ServiceThread;
import java.util.UUID;
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PIDContextTest.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PIDContextTest.java
similarity index 100%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PIDContextTest.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PIDContextTest.java
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PIDController.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PIDController.java
similarity index 100%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PIDController.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PIDController.java
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/ProductSlowerPIDControllerTest.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/ProductSlowerPIDControllerTest.java
similarity index 100%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/ProductSlowerPIDControllerTest.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/ProductSlowerPIDControllerTest.java
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PusherTest.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PusherTest.java
similarity index 96%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PusherTest.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PusherTest.java
index b15db56..4cbcfb4 100644
--- a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PusherTest.java
+++ b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/PusherTest.java
@@ -16,7 +16,7 @@
*/
package org.apache.rocketmq.eventbridge.runtimer.service.backpressure;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.ServiceThread;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.ServiceThread;
import java.text.SimpleDateFormat;
import java.util.Objects;
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/RateEstimatorTest.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/RateEstimatorTest.java
similarity index 97%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/RateEstimatorTest.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/RateEstimatorTest.java
index b8e0f06..a22a663 100644
--- a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/RateEstimatorTest.java
+++ b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/RateEstimatorTest.java
@@ -18,7 +18,7 @@ package org.apache.rocketmq.eventbridge.runtimer.service.backpressure;
import com.google.common.util.concurrent.RateLimiter;
import lombok.SneakyThrows;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.ServiceThread;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.ServiceThread;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
diff --git a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/TransformTest.java b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/TransformTest.java
similarity index 96%
rename from adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/TransformTest.java
rename to adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/TransformTest.java
index 43e7b86..b18e062 100644
--- a/adapter/runtimer/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/TransformTest.java
+++ b/adapter/runtime/src/test/java/org/apache/rocketmq/eventbridge/runtimer/service/backpressure/TransformTest.java
@@ -16,7 +16,7 @@
*/
package org.apache.rocketmq.eventbridge.runtimer.service.backpressure;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.ServiceThread;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.ServiceThread;
import java.text.SimpleDateFormat;
import java.util.Objects;
diff --git a/adapter/runtimer/README.md b/adapter/runtimer/README.md
deleted file mode 100644
index 4e508ba..0000000
--- a/adapter/runtimer/README.md
+++ /dev/null
@@ -1,42 +0,0 @@
-1, Runtimer主线程
-
- 1.1,EventBusListener/SPI
- 订阅事件,转发
- EventBusOnRocketMQListener
- Queue
-
- 1.2 Filter&Transformer
- EventRuleTransfer
- CPU
-
- 1.3 EventTargetPusher
- IO
-
-TODO:
-OpenMessage as base
-Record
-
- MVP edition
-
- Run mvp demo
-
- next meeting: next weekly, 3.5 20:00
-
-TODO:
-metrics, logger, exception
-detail
-
-3.10 Note
-本次会议纪要:
-1、runtimer配置从DB加载;
-2、任务配置的命名:targetRunner;
-3、更新任务配置:从Service修改成API+存储 Watch;
-4、EventBusListener: 不能出现RMQ&Message,只能围绕Connect Recorded;
-5、EventBusListener:支持位点提交;
-
-下次会议时间:3月14日,下周二 19:00
-
-
-
-
-
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/CirculatorContext.java b/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/CirculatorContext.java
deleted file mode 100644
index fd29d8f..0000000
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/CirculatorContext.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import io.openmessaging.connector.api.component.task.sink.SinkTask;
-import io.openmessaging.connector.api.data.ConnectRecord;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.pusher.PusherTaskContext;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.transfer.TransformEngine;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.LoggerName;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetKeyValue;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.enums.RefreshTypeEnum;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.plugin.Plugin;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.plugin.PluginClassLoader;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.config.RuntimerConfigDefine;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-import org.springframework.util.Assert;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.*;
-
-/**
- * event circulator context for transfer and pusher
- */
-@Component
-public class CirculatorContext implements TargetRunnerListener {
-
- private final static Logger logger = LoggerFactory.getLogger(LoggerName.EventBus_Listener);
-
- private BlockingQueue<ConnectRecord> eventQueue = new LinkedBlockingQueue<>(50000);
-
- private BlockingQueue<ConnectRecord> targetQueue = new LinkedBlockingQueue<>(50000);
-
- private Map<String/*RunnerName*/, TransformEngine<ConnectRecord>> taskTransformMap = new ConcurrentHashMap<>(20);
-
- private Map<String/*RunnerName*/, SinkTask> pusherTaskMap = new ConcurrentHashMap<>(20);
-
- private Map<String/*PusherCLass*/, ExecutorService> pusherExecutorMap = new ConcurrentHashMap<>(10);
-
- private Plugin plugin;
-
- public CirculatorContext(Plugin plugin){
- this.plugin = plugin;
- }
-
- /**
- * initial targetRunnerMap, taskTransformMap, pusherTaskMap
- * @param targetRunnerConfigs
- */
- public void initListenerMetadata(Set<TargetRunnerConfig> targetRunnerConfigs) {
- if (CollectionUtils.isEmpty(targetRunnerConfigs)) {
- return;
- }
- for (TargetRunnerConfig targetRunnerConfig : targetRunnerConfigs) {
- onAddTargetRunner(targetRunnerConfig);
- }
- }
-
- @Override
- public void onAddTargetRunner(TargetRunnerConfig targetRunnerConfig) {
- refreshRunnerMetadata(targetRunnerConfig, RefreshTypeEnum.ADD);
- }
-
- @Override
- public void onUpdateTargetRunner(TargetRunnerConfig targetRunnerConfig) {
- refreshRunnerMetadata(targetRunnerConfig, RefreshTypeEnum.UPDATE);
- }
-
- @Override
- public void onDeleteTargetRunner(TargetRunnerConfig targetRunnerConfig) {
- refreshRunnerMetadata(targetRunnerConfig, RefreshTypeEnum.DELETE);
- }
-
- /**
- * offer event records
- *
- * @param connectRecords
- */
- public boolean offerEventRecords(List<ConnectRecord> connectRecords) {
- return eventQueue.addAll(connectRecords);
- }
-
- /**
- * take event record
- *
- * @return
- */
- public ConnectRecord takeEventRecord() {
- if (eventQueue.isEmpty()) {
- return null;
- }
- try {
- return eventQueue.take();
- } catch (Exception exception) {
- logger.error("take event record exception - stack-> ", exception);
- }
- return null;
- }
-
- public Map<String, TransformEngine<ConnectRecord>> getTaskTransformMap() {
- return taskTransformMap;
- }
-
- public Map<String, SinkTask> getPusherTaskMap() {
- return pusherTaskMap;
- }
-
- public boolean offerTargetTaskQueue(List<ConnectRecord> connectRecords) {
- return targetQueue.addAll(connectRecords);
- }
-
- public ConnectRecord takeTargetMap() {
- if (targetQueue.isEmpty()) {
- return null;
- }
- try {
- return targetQueue.take();
- } catch (Exception exception) {
- exception.printStackTrace();
- }
- return null;
- }
-
- /**
- * get specific thread pool by push name
- * @param pushName
- * @return
- */
- public ExecutorService getExecutorService(String pushName){
- return pusherExecutorMap.get(pushName);
- }
-
- /**
- * refresh target runner where config changed
- * @param targetRunnerConfig
- * @param refreshTypeEnum
- */
- private void refreshRunnerMetadata(TargetRunnerConfig targetRunnerConfig, RefreshTypeEnum refreshTypeEnum) {
- String runnerName = targetRunnerConfig.getName();
- switch (refreshTypeEnum) {
- case ADD:
- case UPDATE:
- TransformEngine<ConnectRecord> transformChain = new TransformEngine<>(targetRunnerConfig.getComponents(), plugin);
- taskTransformMap.put(runnerName, transformChain);
-
- int endIndex = targetRunnerConfig.getComponents().size() -1;
- TargetKeyValue targetKeyValue = new TargetKeyValue(targetRunnerConfig.getComponents().get(endIndex));
- SinkTask sinkTask = initTargetSinkTask(targetKeyValue);
- pusherTaskMap.put(runnerName, sinkTask);
-
- String pusherClass = targetKeyValue.getString(RuntimerConfigDefine.RUNNER_CLASS);
- if (StringUtils.isNotEmpty(pusherClass) && !pusherExecutorMap.containsKey(pusherClass)) {
- pusherExecutorMap.put(pusherClass, initDefaultThreadPoolExecutor(pusherClass));
- }
-
- break;
- case DELETE:
- taskTransformMap.remove(runnerName);
- pusherTaskMap.remove(runnerName);
- break;
- default:
- break;
- }
- }
-
- /**
- * init default thread poll param, support auto config
- * @param threadPollName
- * @return
- */
- private ExecutorService initDefaultThreadPoolExecutor(String threadPollName) {
- ThreadFactoryBuilder threadFactory = new ThreadFactoryBuilder().setNameFormat(threadPollName);
- return new ThreadPoolExecutor(200, 300, 1, TimeUnit.SECONDS, new LinkedBlockingQueue<>(300), threadFactory.build());
- }
-
- /**
- * init target sink task
- * @param targetKeyValue
- * @return
- */
- private SinkTask initTargetSinkTask(TargetKeyValue targetKeyValue) {
- String taskClass = targetKeyValue.getString(RuntimerConfigDefine.RUNNER_CLASS);
- ClassLoader loader = plugin.getPluginClassLoader(taskClass);
- Class taskClazz;
- boolean isolationFlag = false;
- try {
- if (loader instanceof PluginClassLoader) {
- taskClazz = ((PluginClassLoader) loader).loadClass(taskClass, false);
- isolationFlag = true;
- } else {
- taskClazz = Class.forName(taskClass);
- }
- SinkTask sinkTask = (SinkTask) taskClazz.getDeclaredConstructor().newInstance();
- sinkTask.init(targetKeyValue);
- PusherTaskContext sinkTaskContext = new PusherTaskContext(targetKeyValue);
- sinkTask.start(sinkTaskContext);
- if (isolationFlag) {
- Plugin.compareAndSwapLoaders(loader);
- }
- return sinkTask;
- }catch (Exception e) {
- e.printStackTrace();
- }
- return null;
- }
-}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/TargetRunnerContext.java b/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/TargetRunnerContext.java
deleted file mode 100644
index 1eff462..0000000
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/boot/listener/TargetRunnerContext.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener;
-
-import com.google.common.collect.Maps;
-import java.util.Map;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
-
-public class TargetRunnerContext implements TargetRunnerListener {
-
- public static Map<String, TargetRunnerConfig> targetRunnerConfigMap = Maps.newHashMap();
-
- @Override
- public void onAddTargetRunner(TargetRunnerConfig targetRunnerConfig) {
-
- }
-
- @Override
- public void onUpdateTargetRunner(TargetRunnerConfig targetRunnerConfig) {
-
- }
-
- @Override
- public void onDeleteTargetRunner(TargetRunnerConfig targetRunnerConfig) {
-
- }
-
- public static TargetRunnerConfig getTargetRunnerConfig(String targetRunnerName) {
- return targetRunnerConfigMap.get(targetRunnerName);
- }
-}
\ No newline at end of file
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfigDefine.java b/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfigDefine.java
deleted file mode 100644
index 578e92a..0000000
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/config/RuntimerConfigDefine.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.rocketmq.eventbridge.adapter.runtimer.config;
-
-import io.openmessaging.connector.api.data.ConnectRecord;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * Define keys for target runner configs.
- */
-public class RuntimerConfigDefine {
-
- /**
- * The full class name of a specific connector implements.
- */
- public static final String CONNECTOR_CLASS = "connector-class";
-
- public static final String RUNNER_CLASS = "class";
-
- public static final String TRANSFER_CLASS = "transfer-class";
-
- public static final String CONNECTOR_DIRECT_ENABLE = "connector-direct-enable";
-
- public static final String TASK_CLASS = "task-class";
-
- public static final String RUNNER_NAME = "runner-name";
-
- public static final String TASK_ID = "task-id";
-
- public static final String TASK_TYPE = "task-type";
-
- public static final String SOURCE_TASK_CLASS = "source-task-class";
-
- public static final String SINK_TASK_CLASS = "sink-task-class";
-
- public static final String MAX_TASK = "max-task";
-
- public static final String CONNECTOR_ID = "connector-id";
-
- /**
- * Last updated time of the configuration.
- */
- public static final String UPDATE_TIMESTAMP = "update-timestamp";
-
- /**
- * Whether the current config is deleted.
- */
- public static final String CONFIG_DELETED = "config-deleted";
-
- /**
- * The full class name of record converter. Which is used to parse {@link ConnectRecord} to/from byte[].
- */
- public static final String SOURCE_RECORD_CONVERTER = "source-record-converter";
-
- public static final String NAMESRV_ADDR = "namesrv-addr";
-
- public static final String RMQ_PRODUCER_GROUP = "rmq-producer-group";
-
- public static final String RMQ_CONSUMNER_GROUP = "rmq-consumer-group";
-
- public static final String OPERATION_TIMEOUT = "operation-timeout";
-
- public static final String HASH_FUNC = "consistentHashFunc";
-
- public static final String VIRTUAL_NODE = "virtualNode";
-
- public static final String CONNECT_SHARDINGKEY = "connect-shardingkey";
-
- public static final String CHANNEL_NAME = "channel-name";
-
- public static final String CONNECT_RULE_NAME = "connect-rule-name";
-
- public static final String CONNECT_TOPICNAMES = "connect-topicnames";
-
- public static final String CONNECT_SOURCE_PARTITION = "connect-source-partition";
-
- public static final String CONNECT_SOURCE_POSITION = "connect-source-position";
-
- public static final String CONNECT_ENTRYTYPE = "connect-entrytype";
-
- public static final String CONNECT_TIMESTAMP = "connect-timestamp";
-
- public static final String CONNECT_SCHEMA = "connect-schema";
-
- public static final String TRANSFORMS = "transforms";
-
- public static final String CONNECT_RECORDS_KEY = "SYSTEM_RETRY_TIMES";
-
- public static final String TARGET_RUNNER_KEY = "eventBusName";
-
-}
diff --git a/adapter/storage/pom.xml b/adapter/storage/pom.xml
index 7cae0f9..f9d11e1 100644
--- a/adapter/storage/pom.xml
+++ b/adapter/storage/pom.xml
@@ -41,7 +41,7 @@
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-eventbridge-adapter-runtimer</artifactId>
+ <artifactId>rocketmq-eventbridge-adapter-runtime</artifactId>
</dependency>
<!-- Framework -->
<dependency>
diff --git a/adapter/storage/src/main/java/org/apache/rocketmq/eventbridge/adapter/storage/rocketmq/impl/RocketMQEventDataRepository.java b/adapter/storage/src/main/java/org/apache/rocketmq/eventbridge/adapter/storage/rocketmq/impl/RocketMQEventDataRepository.java
index d707e25..4e6f64e 100644
--- a/adapter/storage/src/main/java/org/apache/rocketmq/eventbridge/adapter/storage/rocketmq/impl/RocketMQEventDataRepository.java
+++ b/adapter/storage/src/main/java/org/apache/rocketmq/eventbridge/adapter/storage/rocketmq/impl/RocketMQEventDataRepository.java
@@ -90,6 +90,7 @@ public class RocketMQEventDataRepository implements EventDataRepository {
}
@Cacheable(value = "topicCache")
+ @Override
public String getTopicName(String accountId, String eventBusName) {
String topicName = null;
EventTopicDO eventTopicDO = eventTopicMapper.getTopic(accountId, eventBusName);
diff --git a/adapter/storage/src/main/java/org/apache/rocketmq/eventbridge/adapter/storage/rocketmq/runtimer/RocketMQEventSubscriber.java b/adapter/storage/src/main/java/org/apache/rocketmq/eventbridge/adapter/storage/rocketmq/runtimer/RocketMQEventSubscriber.java
index 77941ac..f6b08cd 100644
--- a/adapter/storage/src/main/java/org/apache/rocketmq/eventbridge/adapter/storage/rocketmq/runtimer/RocketMQEventSubscriber.java
+++ b/adapter/storage/src/main/java/org/apache/rocketmq/eventbridge/adapter/storage/rocketmq/runtimer/RocketMQEventSubscriber.java
@@ -20,7 +20,6 @@ package org.apache.rocketmq.eventbridge.adapter.storage.rocketmq.runtimer;
import com.alibaba.fastjson.JSON;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import com.google.gson.Gson;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.data.ConnectRecord;
@@ -37,15 +36,16 @@ import org.apache.rocketmq.client.AccessChannel;
import org.apache.rocketmq.common.UtilAll;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.utils.NetworkUtil;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.boot.listener.EventSubscriber;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.ServiceThread;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.entity.TargetRunnerConfig;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.common.enums.RefreshTypeEnum;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.config.RuntimerConfigDefine;
-import org.apache.rocketmq.eventbridge.adapter.runtimer.service.TargetRunnerConfigObserver;
+import org.apache.rocketmq.eventbridge.adapter.runtime.boot.listener.EventSubscriber;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.ServiceThread;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.entity.SubscribeRunnerKeys;
+import org.apache.rocketmq.eventbridge.adapter.runtime.common.enums.RefreshTypeEnum;
+import org.apache.rocketmq.eventbridge.adapter.runtime.config.RuntimeConfigDefine;
+import org.apache.rocketmq.eventbridge.adapter.runtime.service.TargetRunnerConfigObserver;
import org.apache.rocketmq.eventbridge.adapter.storage.rocketmq.runtimer.consumer.ClientConfig;
import org.apache.rocketmq.eventbridge.adapter.storage.rocketmq.runtimer.consumer.LitePullConsumer;
import org.apache.rocketmq.eventbridge.adapter.storage.rocketmq.runtimer.consumer.LitePullConsumerImpl;
+import org.apache.rocketmq.eventbridge.domain.storage.EventDataRepository;
import org.apache.rocketmq.eventbridge.exception.EventBridgeException;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.proxy.SocksProxyConfig;
@@ -55,19 +55,15 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.support.PropertiesLoaderUtils;
import org.springframework.stereotype.Component;
+import javax.annotation.PostConstruct;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.stream.Collectors;
/**
* RocketMQ implement event subscriber
@@ -77,10 +73,13 @@ public class RocketMQEventSubscriber extends EventSubscriber {
private static final Logger logger = LoggerFactory.getLogger(RocketMQEventSubscriber.class);
- private final BlockingQueue<MessageExt> messageBuffer = new LinkedBlockingQueue<>(50000);
+ @Autowired
+ private EventDataRepository eventDataRepository;
@Autowired
- private final TargetRunnerConfigObserver runnerConfigObserver;
+ private TargetRunnerConfigObserver runnerConfigObserver;
+
+ private final BlockingQueue<MessageExt> messageBuffer = new LinkedBlockingQueue<>(50000);
private Integer pullTimeOut;
private Integer pullBatchSize;
@@ -95,23 +94,23 @@ public class RocketMQEventSubscriber extends EventSubscriber {
private static final String SYS_DEFAULT_GROUP = "event-bridge-default-group";
public static final String QUEUE_OFFSET = "queueOffset";
- private static final String RUNNER_NAME = "runnerName";
+ public static final String MSG_ID = "msgId";
- public RocketMQEventSubscriber(TargetRunnerConfigObserver runnerConfigObserver) {
- this.runnerConfigObserver = runnerConfigObserver;
+ @PostConstruct
+ public void initRocketMQEventSubscriber(){
this.initMqProperties();
- this.initConsumeWorkers(runnerConfigObserver);
+ this.initConsumeWorkers();
}
@Override
- public void refresh(TargetRunnerConfig targetRunnerConfig, RefreshTypeEnum refreshTypeEnum) {
+ public void refresh(SubscribeRunnerKeys subscribeRunnerKeys, RefreshTypeEnum refreshTypeEnum) {
switch (refreshTypeEnum) {
case ADD:
case UPDATE:
- putConsumeWorker(targetRunnerConfig);
+ putConsumeWorker(subscribeRunnerKeys);
break;
case DELETE:
- removeConsumeWorker(targetRunnerConfig);
+ removeConsumeWorker(subscribeRunnerKeys);
break;
default:
break;
@@ -143,25 +142,20 @@ public class RocketMQEventSubscriber extends EventSubscriber {
return connectRecords;
}
- @Override
- public void commit(List<ConnectRecord> connectRecordList) {
- // TODO
- }
-
/**
- * parse topics by specific target runner configs
- * @param targetRunnerConfig
- * @return
+ * group by runner name batch commit
+ * @param connectRecordList
*/
- public Set<String> parseTopicsByRunnerConfig(TargetRunnerConfig targetRunnerConfig){
- Set<String> listenTopics = Sets.newHashSet();
- List<Map<String,String>> runnerConfigMap = targetRunnerConfig.getComponents();
- if (CollectionUtils.isEmpty(runnerConfigMap)){
- logger.warn("target runner config components is empty, config info - {}", targetRunnerConfig);
- return listenTopics;
+ @Override
+ public void commit(List<ConnectRecord> connectRecordList) {
+ if(CollectionUtils.isEmpty(connectRecordList)){
+ logger.warn("commit event record data empty!");
+ return;
}
- listenTopics.add(runnerConfigMap.iterator().next().get(RuntimerConfigDefine.CHANNEL_NAME));
- return listenTopics;
+ String runnerName = connectRecordList.iterator().next().getExtension(RuntimeConfigDefine.RUNNER_NAME);
+ List<String> msgIds = connectRecordList.stream().map(item -> item.getPosition()
+ .getPartition().getPartition().get(MSG_ID).toString()).collect(Collectors.toList());
+ consumeWorkerMap.get(runnerName).commit(msgIds);
}
/**
@@ -170,7 +164,7 @@ public class RocketMQEventSubscriber extends EventSubscriber {
private void initMqProperties() {
try {
ClientConfig clientConfig = new ClientConfig();
- Properties properties = PropertiesLoaderUtils.loadAllProperties("runtimer.properties");
+ Properties properties = PropertiesLoaderUtils.loadAllProperties("runtime.properties");
String namesrvAddr = properties.getProperty("rocketmq.namesrvAddr");
String consumerGroup = properties.getProperty("rocketmq.consumerGroup");
pullTimeOut = Integer.valueOf(properties.getProperty("rocketmq.consumer.pullTimeOut"));
@@ -214,40 +208,40 @@ public class RocketMQEventSubscriber extends EventSubscriber {
/**
* init rocket mq pull consumer
*/
- private void initConsumeWorkers(TargetRunnerConfigObserver runnerConfigObserver) {
- for (TargetRunnerConfig targetRunnerConfig : runnerConfigObserver.getTargetRunnerConfig()) {
- LitePullConsumer litePullConsumer = initLitePullConsumer(targetRunnerConfig);
- ConsumeWorker consumeWorker = new ConsumeWorker(litePullConsumer, targetRunnerConfig.getName());
- consumeWorkerMap.put(targetRunnerConfig.getName(), consumeWorker);
+ private void initConsumeWorkers() {
+ for (SubscribeRunnerKeys subscribeRunnerKeys : runnerConfigObserver.getSubscribeRunnerKeys()) {
+ LitePullConsumer litePullConsumer = initLitePullConsumer(subscribeRunnerKeys);
+ ConsumeWorker consumeWorker = new ConsumeWorker(litePullConsumer, subscribeRunnerKeys.getRunnerName());
+ consumeWorkerMap.put(subscribeRunnerKeys.getRunnerName(), consumeWorker);
consumeWorker.start();
}
-
}
/**
* first init default rocketmq pull consumer
* @return
*/
- public LitePullConsumer initLitePullConsumer(TargetRunnerConfig targetRunnerConfig) {
- Set<String> topics = parseTopicsByRunnerConfig(targetRunnerConfig);
-
+ public LitePullConsumer initLitePullConsumer(SubscribeRunnerKeys subscribeRunnerKeys) {
+ String topic = getTopicName(subscribeRunnerKeys);
RPCHook rpcHook = this.sessionCredentials != null ? new AclClientRPCHook(this.sessionCredentials) : null;
LitePullConsumerImpl pullConsumer = new LitePullConsumerImpl(this.clientConfig, rpcHook);
if (StringUtils.isNotBlank(this.socksProxy)) {
pullConsumer.setSockProxyJson(this.socksProxy);
}
try {
- for(String topic : topics){
- pullConsumer.attachTopic(topic, "*");
- }
+ pullConsumer.attachTopic(topic, "*");
pullConsumer.startup();
} catch (Exception exception) {
- logger.error("init default pull consumer exception, topic -" + topics.toString() + "-stackTrace-", exception);
+ logger.error("init default pull consumer exception, topic -" + topic + "-stackTrace-", exception);
throw new EventBridgeException(" init rocketmq consumer failed");
}
return pullConsumer;
}
+ private String getTopicName(SubscribeRunnerKeys subscribeRunnerKeys) {
+ return eventDataRepository.getTopicName(subscribeRunnerKeys.getAccountId(), subscribeRunnerKeys.getEventBusName());
+ }
+
private String createGroupName(String prefix) {
StringBuilder sb = new StringBuilder();
sb.append(prefix).append("-");
@@ -267,17 +261,16 @@ public class RocketMQEventSubscriber extends EventSubscriber {
Schema schema;
Long timestamp;
ConnectRecord sinkRecord;
- String connectTimestamp = properties.get(RuntimerConfigDefine.CONNECT_TIMESTAMP);
+ String connectTimestamp = properties.get(RuntimeConfigDefine.CONNECT_TIMESTAMP);
timestamp = StringUtils.isNotEmpty(connectTimestamp) ? Long.valueOf(connectTimestamp) : null;
- String connectSchema = properties.get(RuntimerConfigDefine.CONNECT_SCHEMA);
+ String connectSchema = properties.get(RuntimeConfigDefine.CONNECT_SCHEMA);
schema = StringUtils.isNotEmpty(connectSchema) ? JSON.parseObject(connectSchema, Schema.class) : null;
byte[] body = messageExt.getBody();
- RecordPartition recordPartition = convertToRecordPartition(messageExt.getTopic(), messageExt.getBrokerName(), messageExt.getQueueId());
+ RecordPartition recordPartition = convertToRecordPartition(messageExt.getTopic(), messageExt.getBrokerName(), messageExt.getQueueId(), messageExt.getMsgId());
RecordOffset recordOffset = convertToRecordOffset(messageExt.getQueueOffset());
String bodyStr = new String(body, StandardCharsets.UTF_8);
sinkRecord = new ConnectRecord(recordPartition, recordOffset, timestamp, schema, bodyStr);
KeyValue keyValue = new DefaultKeyValue();
- keyValue.put(RuntimerConfigDefine.CHANNEL_NAME, messageExt.getTopic());
if (MapUtils.isNotEmpty(properties)) {
for (Map.Entry<String, String> entry : properties.entrySet()) {
keyValue.put(entry.getKey(), entry.getValue());
@@ -287,11 +280,12 @@ public class RocketMQEventSubscriber extends EventSubscriber {
return sinkRecord;
}
- private RecordPartition convertToRecordPartition(String topic, String brokerName, int queueId) {
+ private RecordPartition convertToRecordPartition(String topic, String brokerName, int queueId, String msgId) {
Map<String, String> map = new HashMap<>();
map.put("topic", topic);
map.put("brokerName", brokerName);
map.put("queueId", queueId + "");
+ map.put(MSG_ID, msgId);
RecordPartition recordPartition = new RecordPartition(map);
return recordPartition;
}
@@ -303,19 +297,19 @@ public class RocketMQEventSubscriber extends EventSubscriber {
return recordOffset;
}
- private void putConsumeWorker(TargetRunnerConfig targetRunnerConfig) {
- ConsumeWorker consumeWorker = consumeWorkerMap.get(targetRunnerConfig.getName());
+ private void putConsumeWorker(SubscribeRunnerKeys subscribeRunnerKeys) {
+ ConsumeWorker consumeWorker = consumeWorkerMap.get(subscribeRunnerKeys.getRunnerName());
if (!Objects.isNull(consumeWorker)){
consumeWorker.shutdown();
}
- LitePullConsumer litePullConsumer = initLitePullConsumer(targetRunnerConfig);
- ConsumeWorker newWorker = new ConsumeWorker(litePullConsumer, targetRunnerConfig.getName());
- consumeWorkerMap.put(targetRunnerConfig.getName(), newWorker);
+ LitePullConsumer litePullConsumer = initLitePullConsumer(subscribeRunnerKeys);
+ ConsumeWorker newWorker = new ConsumeWorker(litePullConsumer, subscribeRunnerKeys.getRunnerName());
+ consumeWorkerMap.put(subscribeRunnerKeys.getRunnerName(), newWorker);
newWorker.start();
}
- private void removeConsumeWorker(TargetRunnerConfig targetRunnerConfig) {
- ConsumeWorker consumeWorker = consumeWorkerMap.remove(targetRunnerConfig.getName());
+ private void removeConsumeWorker(SubscribeRunnerKeys subscribeRunnerKeys) {
+ ConsumeWorker consumeWorker = consumeWorkerMap.remove(subscribeRunnerKeys.getRunnerName());
if (!Objects.isNull(consumeWorker)){
consumeWorker.shutdown();
}
@@ -342,7 +336,7 @@ public class RocketMQEventSubscriber extends EventSubscriber {
try {
List<MessageExt> messages = pullConsumer.poll(pullBatchSize, Duration.ofSeconds(pullTimeOut));
for (MessageExt message : messages) {
- message.putUserProperty(RUNNER_NAME, runnerName);
+ message.putUserProperty(RuntimeConfigDefine.RUNNER_NAME, runnerName);
messageBuffer.put(message);
}
} catch (Exception exception) {
@@ -351,6 +345,10 @@ public class RocketMQEventSubscriber extends EventSubscriber {
}
}
+ public void commit(List<String> messageIds){
+ this.pullConsumer.commit(messageIds);
+ }
+
@Override
public void shutdown() {
pullConsumer.shutdown();
diff --git a/domain/src/main/java/org/apache/rocketmq/eventbridge/domain/storage/EventDataRepository.java b/domain/src/main/java/org/apache/rocketmq/eventbridge/domain/storage/EventDataRepository.java
index de87103..55a412e 100644
--- a/domain/src/main/java/org/apache/rocketmq/eventbridge/domain/storage/EventDataRepository.java
+++ b/domain/src/main/java/org/apache/rocketmq/eventbridge/domain/storage/EventDataRepository.java
@@ -52,4 +52,11 @@ public interface EventDataRepository {
*/
String getEventBusPersistentContext(String accountId, String eventBusName);
+ /**
+ * @param accountId
+ * @param eventBusName
+ * @return
+ */
+ String getTopicName(String accountId, String eventBusName);
+
}
diff --git a/infrastructure/README-CN.md b/infrastructure/README-CN.md
new file mode 100644
index 0000000..7022b24
--- /dev/null
+++ b/infrastructure/README-CN.md
@@ -0,0 +1,16 @@
+## infrastructure
+
+* 含义:基础设施层
+* 作用:存放EventBridge所依赖的基础设施组件
+
+## 核心组件有下述内容
+
+## 1, metric
+
+## 2, trace
+
+## 3, logger
+
+## 4, auth
+
+## 5, monitor
diff --git a/adapter/pom.xml b/infrastructure/pom.xml
similarity index 70%
copy from adapter/pom.xml
copy to infrastructure/pom.xml
index ab600bb..d419a45 100644
--- a/adapter/pom.xml
+++ b/infrastructure/pom.xml
@@ -9,24 +9,21 @@
OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>rocketmq-eventbridge</artifactId>
<groupId>org.apache.rocketmq</groupId>
<version>1.0.0</version>
- <relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <artifactId>rocketmq-eventbridge-adapter</artifactId>
- <packaging>pom</packaging>
- <modules>
- <module>persistence</module>
- <module>api</module>
- <module>rpc</module>
- <module>runtimer</module>
- <module>storage</module>
- </modules>
+ <artifactId>rocketmq-eventbridge-infrastructure</artifactId>
+
+ <properties>
+ <maven.compiler.source>8</maven.compiler.source>
+ <maven.compiler.target>8</maven.compiler.target>
+ </properties>
</project>
\ No newline at end of file
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceFactory.java b/infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/metric/MonitorFactory.java
similarity index 91%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceFactory.java
rename to infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/metric/MonitorFactory.java
index f4f3b7d..6311d2a 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceFactory.java
+++ b/infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/metric/MonitorFactory.java
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.trace;
+package org.apache.rocketmq.eventbridge.infrastructure.metric;
import java.util.Map;
-public class TraceFactory {
+public class MonitorFactory {
public void createSpan(Map<String,String> content){
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceException.java b/infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/TraceException.java
similarity index 93%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceException.java
rename to infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/TraceException.java
index 6baab11..6db0190 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceException.java
+++ b/infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/TraceException.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.trace;
+package org.apache.rocketmq.eventbridge.infrastructure.trace;
public class TraceException extends Exception{
}
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceStrategy.java b/infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/TraceStrategy.java
similarity index 94%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceStrategy.java
rename to infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/TraceStrategy.java
index 4020ac3..739ecbe 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/TraceStrategy.java
+++ b/infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/TraceStrategy.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.trace;
+package org.apache.rocketmq.eventbridge.infrastructure.trace;
/**
diff --git a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/ZipkinTraceStrategy.java b/infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/ZipkinTraceStrategy.java
similarity index 94%
rename from adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/ZipkinTraceStrategy.java
rename to infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/ZipkinTraceStrategy.java
index 7c0d6a9..d8a7676 100644
--- a/adapter/runtimer/src/main/java/org/apache/rocketmq/eventbridge/adapter/runtimer/trace/ZipkinTraceStrategy.java
+++ b/infrastructure/src/main/java/org/apache/rocketmq/eventbridge/infrastructure/trace/ZipkinTraceStrategy.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.rocketmq.eventbridge.adapter.runtimer.trace;
+package org.apache.rocketmq.eventbridge.infrastructure.trace;
public class ZipkinTraceStrategy implements TraceStrategy {
@Override
diff --git a/pom.xml b/pom.xml
index 2660d67..a855458 100644
--- a/pom.xml
+++ b/pom.xml
@@ -102,6 +102,7 @@
<module>adapter</module>
<module>domain</module>
<module>common</module>
+ <module>infrastructure</module>
</modules>
<dependencyManagement>
@@ -134,7 +135,7 @@
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-eventbridge-adapter-runtimer</artifactId>
+ <artifactId>rocketmq-eventbridge-adapter-runtime</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -142,6 +143,11 @@
<artifactId>rocketmq-eventbridge-adapter-storage</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-eventbridge-infrastructure</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<!-- Framework -->
<dependency>
diff --git a/start/pom.xml b/start/pom.xml
index 963446e..2bf80b0 100644
--- a/start/pom.xml
+++ b/start/pom.xml
@@ -40,7 +40,7 @@
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-eventbridge-adapter-runtimer</artifactId>
+ <artifactId>rocketmq-eventbridge-adapter-runtime</artifactId>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
@@ -50,6 +50,10 @@
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-eventbridge-domain</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-eventbridge-infrastructure</artifactId>
+ </dependency>
<!-- Framework -->
<dependency>
<groupId>org.springframework.boot</groupId>
diff --git a/start/src/main/resources/application.properties b/start/src/main/resources/application.properties
index 3f86877..7ef7c9f 100644
--- a/start/src/main/resources/application.properties
+++ b/start/src/main/resources/application.properties
@@ -29,11 +29,12 @@ spring.flyway.placeholderReplacement=false
rocketmq.namesrvAddr=localhost:9876
rocketmq.connect.endpoint=http://127.0.0.1:8082
rocketmq.cluster.name=DefaultCluster
-## runtimer
-rumtimer.config.mode=FILE
-rumtimer.name=eventbridge-runtimer
-runtimer.pluginpath=/Users/Local/eventbridge/plugin
-runtimer.storePathRootDir=/Users/Local/eventbridge/store
+## runtime
+runtime.config.mode=FILE
+runtime.storage.mode=ROCKETMQ
+rumtime.name=eventbridge-runtimer
+runtime.pluginpath=/Users/Local/eventbridge/plugin
+runtime.storePathRootDir=/Users/Local/eventbridge/store
## log
app.name=rocketmqeventbridge