You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by zh...@apache.org on 2022/03/02 03:39:10 UTC
[rocketmq-connect] branch master updated (ce39378 -> 258e37e)
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git.
from ce39378 Add 'connector/rocketmq-connect-redis/' from commit '4fdd8b2e20c4d815e7bd5c9cef13be89fcd56c02'
new 952aa8f (1)Rename rocketmq-connector to replicator (2) Initialize RocketMQ activemq connect and runtime
new 208e4ed init commit (#309)
new 6ddd7e4 Define and Implement the RmqConnector and RmqSourceTask. (#343)
new c7a2c41 [ISSUE #351] rmq client setInstanceName on sourceTask (#355)
new 25e699e return taskConfig with source-record-converter (#373)
new c1a51cc 【ISSUE #278】Define and Implement the RmqConnector and RmqSourceTask. (#381)
new 40b9fc8 [ISSUE 368]Polish rocketmq replicator implementation (#366)
new ef2a365 Update README.md
new d80142a Support wildcard subscription topic. resolve #395
new b4f8cd9 Automatically create target topic. resolve #396
new 81376b9 Topic rename. resolve #399
new ad38ec7 extend messageQueue for TaskTopicInfo
new 90541a2 add DivideTaskByQueue. resolve #397
new d4e677d fix sourceTaskTopicList is null. resolve #386
new 6809200 Merge pull request #422 from xujianhai666/fix-emptyTopic
new ebb44c1 Increase topic change awareness monitoring. resolve #398
new caea266 Merge pull request #424 from xujianhai666/feat-queuelistener
new dfb32fd chore(runtime/replicator): change rocketmq dependency from 4.4.0 to 4.5.2
new a321762 feat(replicator):add consumer offset sync
new d7b0ce8 feat(replicator): Support subcriptionConfig sync
new b3377a6 [ISSUES #434] Replicator support RocketMQConverter (#463)
new 2d00797 feat(replicator): support DivideTaskByConsistentHash (#443)
new ff09b42 feat(replicator): Add commitRecord after producer send success (#452)
new 6182624 When replicator stop, stop MQAdminExt and Listener (#474)
new 5ed354b The topic name in the TaskTopicInfo constructor should be topic (#473)
new 7201d00 [ISSUE #468] Replicator is packaged as fatjar (#472)
new f55d213 [ISSUE #478] TopicList is null exception and frequent requestTaskReconfiguration (#483)
new 070dd98 [ISSUE #492] Optimize metadata synchronization and fix RocketMQConverter bug (#493)
new ae2751e [ISSUE 503] Metadata synchronization optimization (#504)
new d2e168e Supplemental replicator documentation (#506)
new b0ca555 docs(replicator): add introduction of replicator (#507)
new 2e8ea2e Update README.md
new 8e66fd6 [Replicator] Fix message duplication problem (#692)
new a0b10aa [ISSUE #699] [Replicator] The source task starts to check the position (#700)
new 1ca3a07 [rocketmq-replicator] Support ACL (#832)
new 691ff01 [rocketmq-replicator] Bug fix: Missing subscription group when syncing offset (#837)
new d1845c5 [rocketmq-replicator] Fix topic build route logic (#834)
new 55631bb [rocketmq-replicator] Support subGroup whitelist (#843)
new 258e37e Add 'connector/rocketmq-replicator/' from commit '55631bb1fe4d3864c2d45dc7fee9e49764d7b3ac'
The 39 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
connector/rocketmq-replicator/.gitignore | 13 +
connector/rocketmq-replicator/README.md | 69 +++++
connector/rocketmq-replicator/package.xml | 41 +++
connector/rocketmq-replicator/pom.xml | 154 ++++++++++
.../apache/rocketmq/replicator/MetaSourceTask.java | 156 ++++++++++
.../apache/rocketmq/replicator/RmqConstants.java | 20 +-
.../rocketmq/replicator/RmqMetaReplicator.java | 254 ++++++++++++++++
.../rocketmq/replicator/RmqSourceReplicator.java | 333 +++++++++++++++++++++
.../apache/rocketmq/replicator/RmqSourceTask.java | 273 +++++++++++++++++
.../rocketmq/replicator}/common/ConstDefine.java | 12 +-
.../apache/rocketmq/replicator/common/Utils.java | 203 +++++++++++++
.../rocketmq/replicator/config/ConfigDefine.java | 68 +++++
.../rocketmq/replicator}/config/ConfigUtil.java | 2 +-
.../rocketmq/replicator}/config/DataType.java | 2 +-
.../replicator/config/RmqConnectorConfig.java | 172 +++++++++++
.../rocketmq/replicator/config/TaskConfig.java | 133 ++++++++
.../rocketmq/replicator/config/TaskConfigEnum.java | 48 +++
.../replicator/config/TaskDivideConfig.java | 123 ++++++++
.../rocketmq/replicator}/config/TaskTopicInfo.java | 6 +-
.../rocketmq/replicator/offset/OffsetSync.java | 75 +++++
.../replicator/offset/OffsetSyncStore.java | 90 ++++++
.../rocketmq/replicator/schema/FieldName.java} | 18 +-
.../replicator}/strategy/DivideStrategyEnum.java | 2 +-
.../strategy/DivideTaskByConsistentHash.java | 89 ++++++
.../replicator}/strategy/DivideTaskByQueue.java | 37 +--
.../replicator/strategy/DivideTaskByTopic.java | 60 ++++
.../replicator}/strategy/TaskDivideStrategy.java | 13 +-
.../replicator/RmqSourceReplicatorTest.java | 66 ++++
28 files changed, 2479 insertions(+), 53 deletions(-)
create mode 100644 connector/rocketmq-replicator/.gitignore
create mode 100644 connector/rocketmq-replicator/README.md
create mode 100644 connector/rocketmq-replicator/package.xml
create mode 100644 connector/rocketmq-replicator/pom.xml
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
copy rocketmq-connect-sample/src/main/java/org/apache/rocketmq/connect/file/FileConstants.java => connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java (55%)
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
copy connector/{rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc => rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator}/common/ConstDefine.java (70%)
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
copy connector/{rocketmq-connect-kafka/src/main/java/org/apache/rocketmq/connect/kafka => rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator}/config/ConfigUtil.java (98%)
copy connector/{rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc => rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator}/config/DataType.java (94%)
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
copy connector/{rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc => rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator}/config/TaskTopicInfo.java (85%)
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
copy connector/{rocketmq-connect-rabbitmq/src/test/java/org/apache/rocketmq/connect/jms/RabbitmqConfigTest.java => rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java} (75%)
copy connector/{rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc => rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator}/strategy/DivideStrategyEnum.java (94%)
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
copy connector/{rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc => rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator}/strategy/DivideTaskByQueue.java (61%)
create mode 100644 connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
copy connector/{rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc => rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator}/strategy/TaskDivideStrategy.java (70%)
create mode 100644 connector/rocketmq-replicator/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
[rocketmq-connect] 24/39: When replicator stop, stop MQAdminExt and Listener (#474)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 61826240f53dfdf2fa9690d5b89a9ed2fc35d41f
Author: zhoubo <87...@qq.com>
AuthorDate: Fri Nov 29 21:28:53 2019 +0800
When replicator stop, stop MQAdminExt and Listener (#474)
* https://github.com/apache/rocketmq-externals/issues/469
---
.../java/org/apache/rocketmq/replicator/RmqSourceReplicator.java | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index c560840..e868b89 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -170,17 +170,24 @@ public class RmqSourceReplicator extends SourceConnector {
return true;
}
+ @Override
public void stop() {
+ executor.shutdown();
+ this.srcMQAdminExt.shutdown();
+ this.targetMQAdminExt.shutdown();
}
+ @Override
public void pause() {
}
+ @Override
public void resume() {
}
+ @Override
public Class<? extends Task> taskClass() {
return RmqSourceTask.class;
[rocketmq-connect] 22/39: feat(replicator): support DivideTaskByConsistentHash (#443)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 2d0079795fb68050cf59d9946a597d0116fb0139
Author: xujianhai666 <52...@users.noreply.github.com>
AuthorDate: Tue Nov 19 23:25:02 2019 +0800
feat(replicator): support DivideTaskByConsistentHash (#443)
- Add assign by consistentHash pattern
Closes #439
---
...yQueue.java => DivideTaskByConsistentHash.java} | 62 +++++++++++++++-------
.../replicator/strategy/DivideTaskByQueue.java | 1 -
.../replicator/strategy/TaskDivideStrategy.java | 1 -
3 files changed, 43 insertions(+), 21 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
similarity index 54%
copy from src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
copy to src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
index d909873..b027246 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
@@ -1,12 +1,12 @@
-/*
+package org.apache.rocketmq.replicator.strategy;/*
* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
+ * contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,35 +14,44 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.replicator.strategy;
import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.internal.DefaultKeyValue;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
-import org.apache.rocketmq.common.message.MessageQueue;
+import java.util.List;
+import java.util.Map;
+import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter;
+import org.apache.rocketmq.common.consistenthash.Node;
import org.apache.rocketmq.replicator.config.DataType;
import org.apache.rocketmq.replicator.config.TaskConfigEnum;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
import org.apache.rocketmq.replicator.config.TaskTopicInfo;
-public class DivideTaskByQueue extends TaskDivideStrategy {
- public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
+public class DivideTaskByConsistentHash extends TaskDivideStrategy {
+ @Override public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
- List<KeyValue> config = new ArrayList<KeyValue>();
+ List<KeyValue> config = new ArrayList<>();
int parallelism = tdc.getTaskParallelism();
- Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
+ Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
int id = -1;
- for (String t : topicRouteMap.keySet()) {
- for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
- int ind = ++id % parallelism;
- if (!queueTopicList.containsKey(ind)) {
- queueTopicList.put(ind, new ArrayList<TaskTopicInfo>());
+
+ Collection<ClientNode> cidNodes = new ArrayList<>();
+ for (int i = 0; i < parallelism; i++) {
+ cidNodes.add(new ClientNode(i, Integer.toString(i)));
+ queueTopicList.put(i, new ArrayList<>());
+ }
+
+ ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
+
+ for (String t : topicMap.keySet()) {
+ for (TaskTopicInfo queue : topicMap.get(t)) {
+ ClientNode clientNode = router.routeNode(queue.toString());
+ if (clientNode != null) {
+ queueTopicList.get(clientNode.index).add(queue);
}
- queueTopicList.get(ind).add(taskTopicInfo);
}
}
@@ -58,4 +67,19 @@ public class DivideTaskByQueue extends TaskDivideStrategy {
return config;
}
+
+ private static class ClientNode implements Node {
+ private final String clientID;
+ private final int index;
+
+ public ClientNode(int index, String clientID) {
+ this.index = index;
+ this.clientID = clientID;
+ }
+
+ @Override
+ public String getKey() {
+ return clientID;
+ }
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
index d909873..0a493de 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
@@ -20,7 +20,6 @@ import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.internal.DefaultKeyValue;
import java.util.HashMap;
-import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.replicator.config.DataType;
import org.apache.rocketmq.replicator.config.TaskConfigEnum;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java b/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
index 093d92e..6f58bb3 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
@@ -17,7 +17,6 @@
package org.apache.rocketmq.replicator.strategy;
import io.openmessaging.KeyValue;
-import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import java.util.List;
import java.util.Map;
[rocketmq-connect] 04/39: [ISSUE #351] rmq client setInstanceName on sourceTask (#355)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit c7a2c41c5abfafe41d89b912452791787e11539a
Author: SanChen <sa...@chenyiliu.com>
AuthorDate: Wed Aug 7 11:00:16 2019 +0800
[ISSUE #351] rmq client setInstanceName on sourceTask (#355)
---
.../apache/rocketmq/connector/RmqSourceConnector.java | 3 ++-
.../org/apache/rocketmq/connector/RmqSourceTask.java | 1 +
.../java/org/apache/rocketmq/connector/common/Utils.java | 16 ++++++++++++++++
3 files changed, 19 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java b/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
index 2b5bda4..33eb958 100644
--- a/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
@@ -111,7 +111,8 @@ public class RmqSourceConnector extends SourceConnector {
RPCHook rpcHook = null;
this.defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
this.defaultMQAdminExt.setNamesrvAddr(this.config.getString(ConfigDefine.CONN_SOURCE_RMQ));
- this.defaultMQAdminExt.setInstanceName(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ this.defaultMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ this.defaultMQAdminExt.setInstanceName(Utils.createInstanceName(this.config.getString(ConfigDefine.CONN_SOURCE_RMQ)));
try {
defaultMQAdminExt.start();
} catch (MQClientException e) {
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
index 4fca698..7b3b011 100644
--- a/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
@@ -71,6 +71,7 @@ public class RmqSourceTask extends SourceTask {
ConfigUtil.load(config, this.config);
this.consumer.setConsumerGroup(this.taskId);
this.consumer.setNamesrvAddr(this.config.getSourceRocketmq());
+ this.consumer.setInstanceName(Utils.createInstanceName(this.config.getSourceRocketmq()));
List<TaskTopicInfo> topicList = JSONObject.parseArray(this.config.getTaskTopicList(), TaskTopicInfo.class);
try {
diff --git a/src/main/java/org/apache/rocketmq/connector/common/Utils.java b/src/main/java/org/apache/rocketmq/connector/common/Utils.java
index f5822c6..30fe44c 100644
--- a/src/main/java/org/apache/rocketmq/connector/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/connector/common/Utils.java
@@ -16,6 +16,10 @@
*/
package org.apache.rocketmq.connector.common;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
public class Utils {
public static String createGroupName(String prefix) {
@@ -29,4 +33,16 @@ public class Utils {
public static String createTaskId(String prefix) {
return new StringBuilder().append(prefix).append("@").append(System.currentTimeMillis()).toString();
}
+
+ public static String createInstanceName(String namesrvAddr) {
+ String[] namesrvArray = namesrvAddr.split(";");
+ List<String> namesrvList = new ArrayList<String>();
+ for (String ns: namesrvArray) {
+ if (!namesrvList.contains(ns)) {
+ namesrvList.add(ns);
+ }
+ }
+ Collections.sort(namesrvList);
+ return String.valueOf(namesrvList.toString().hashCode());
+ }
}
[rocketmq-connect] 02/39: init commit (#309)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 208e4ed922ddb19efb56b3d924f9c870fcc8a322
Author: chuenfaiy <ch...@163.com>
AuthorDate: Fri Jul 12 21:33:26 2019 +0800
init commit (#309)
---
.gitignore | 14 ++
pom.xml | 79 +++++++++
.../apache/rocketmq/connector/RmqConstants.java | 32 ++++
.../rocketmq/connector/RmqSourceConnector.java | 129 +++++++++++++++
.../apache/rocketmq/connector/RmqSourceTask.java | 182 +++++++++++++++++++++
.../apache/rocketmq/connector/common/Utils.java | 28 ++++
.../rocketmq/connector/config/ConfigDefine.java | 48 ++++++
.../rocketmq/connector/config/ConfigUtil.java | 70 ++++++++
.../apache/rocketmq/connector/config/DataType.java | 25 +++
.../rocketmq/connector/config/TaskConfig.java | 88 ++++++++++
.../rocketmq/connector/schema/FieldName.java | 31 ++++
.../connector/strategy/DivideStrategyEnum.java | 23 +++
.../connector/strategy/DivideTaskByQueue.java | 51 ++++++
.../connector/strategy/DivideTaskByTopic.java | 49 ++++++
.../connector/strategy/TaskDivideStrategy.java | 28 ++++
15 files changed, 877 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..36716aa
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,14 @@
+*.iml
+*.class
+*.jar
+*dependency-reduced-pom.xml
+.classpath
+.project
+.settings/
+target/
+devenv
+*.log*
+*.iml
+.idea/
+*.versionsBackup
+.DS_Store
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 0c79eee..13a5af4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,4 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -8,5 +25,67 @@
<artifactId>rocketmq-connector</artifactId>
<version>1.0-SNAPSHOT</version>
+ <properties>
+ <rocketmq.version>4.4.0</rocketmq.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-connector</artifactId>
+ <version>0.1.0-beta</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.5</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.0.13</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-tools</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-remoting</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jooq</groupId>
+ <artifactId>joor</artifactId>
+ <version>0.9.6</version>
+ </dependency>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-connect</artifactId>
+ <version>0.1.0-beta</version>
+ </dependency>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-connector</artifactId>
+ <version>0.1.0-beta</version>
+ </dependency>
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>fastjson</artifactId>
+ <version>1.2.51</version>
+ </dependency>
+ </dependencies>
</project>
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqConstants.java b/src/main/java/org/apache/rocketmq/connector/RmqConstants.java
new file mode 100644
index 0000000..1278424
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/RmqConstants.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector;
+
+public class RmqConstants {
+
+ public static final String BROKER_NAME = "brokerName";
+
+ public static final String TOPIC_NAME = "topic";
+
+ public static final String QUEUE_ID = "queueId";
+
+ public static final String NEXT_POSITION = "nextPosition";
+
+ public static String getPartition(String topic, String broker, String queueId) {
+ return new StringBuilder().append(broker).append(topic).append(queueId).toString();
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java b/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
new file mode 100644
index 0000000..c919fa3
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import io.openmessaging.connector.api.source.SourceConnector;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.constant.PermName;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.body.TopicList;
+import org.apache.rocketmq.common.protocol.route.QueueData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.connector.config.ConfigDefine;
+import org.apache.rocketmq.connector.strategy.DivideStrategyEnum;
+import org.apache.rocketmq.connector.strategy.DivideTaskByQueue;
+import org.apache.rocketmq.connector.strategy.DivideTaskByTopic;
+import org.apache.rocketmq.connector.strategy.TaskDivideStrategy;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RmqSourceConnector extends SourceConnector {
+
+ private static final Logger log = LoggerFactory.getLogger(RmqSourceConnector.class);
+
+ private KeyValue config;
+
+ private Map<String, List<MessageQueue>> topicRouteMap;
+
+ private final TaskDivideStrategy taskDivideStrategy;
+
+ public RmqSourceConnector() {
+
+ topicRouteMap = new HashMap<String, List<MessageQueue>>();
+
+ if (this.config.getInt(ConfigDefine.TASK_DIVIDE_STRATEGY) == DivideStrategyEnum.BY_TOPIC.ordinal()) {
+ taskDivideStrategy = new DivideTaskByTopic();
+ } else {
+ taskDivideStrategy = new DivideTaskByQueue();
+ }
+ }
+
+ public String verifyAndSetConfig(KeyValue config) {
+
+ for(String requestKey : ConfigDefine.REQUEST_CONFIG){
+ if(!config.containsKey(requestKey)){
+ return "Request config key: " + requestKey;
+ }
+ }
+ this.config = config;
+ return "";
+ }
+
+ public void start() {
+ }
+
+ public void stop() {
+
+ }
+
+ public void pause() {
+
+ }
+
+ public void resume() {
+
+ }
+
+ public Class<? extends Task> taskClass() {
+ return null;
+ }
+
+ public List<KeyValue> taskConfigs() {
+
+ System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, this.config.getString(ConfigDefine.SOURCE_RMQ));
+ RPCHook rpcHook = null;
+ DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
+ try {
+ defaultMQAdminExt.start();
+ TopicList topicList = defaultMQAdminExt.fetchAllTopicList();
+ for (String topic: topicList.getTopicList()) {
+ if (!topic.equals(ConfigDefine.STORE_TOPIC)) {
+ TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+ if (!topicRouteMap.containsKey(topic)) {
+ topicRouteMap.put(topic, new ArrayList<MessageQueue>());
+ }
+ for (QueueData qd: topicRouteData.getQueueDatas()) {
+ if (PermName.isReadable(qd.getPerm())) {
+ for (int i = 0; i < qd.getReadQueueNums(); i++) {
+ MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
+ topicRouteMap.get(topic).add(mq);
+ }
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("fetch topic list error: ", e);
+ } finally {
+ defaultMQAdminExt.shutdown();
+ }
+
+ return this.taskDivideStrategy.divide(this.topicRouteMap,
+ this.config.getString(ConfigDefine.SOURCE_RMQ), this.config.getString(ConfigDefine.STORE_TOPIC));
+ }
+}
+
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
new file mode 100644
index 0000000..d71dc87
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.data.*;
+import io.openmessaging.connector.api.source.SourceTask;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.connector.common.Utils;
+import org.apache.rocketmq.connector.config.ConfigUtil;
+import org.apache.rocketmq.connector.config.DataType;
+import org.apache.rocketmq.connector.config.TaskConfig;
+import org.apache.rocketmq.connector.schema.FieldName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+
+public class RmqSourceTask extends SourceTask {
+
+ private static final Logger log = LoggerFactory.getLogger(RmqSourceTask.class);
+
+ private final String taskId;
+ private final TaskConfig config;
+ private final DefaultMQPullConsumer consumer;
+
+ private Map<MessageQueue, Long> mqOffsetMap;
+ public RmqSourceTask() {
+ this.config = new TaskConfig();
+ this.consumer = new DefaultMQPullConsumer();
+ this.taskId = Utils.createTaskId(Thread.currentThread().getName());
+ mqOffsetMap = new HashMap<MessageQueue, Long>();
+ }
+
+ public Collection<SourceDataEntry> poll() {
+
+ if (this.config.getDataType() == DataType.COMMON_MESSAGE.ordinal()) {
+ return pollCommonMessage();
+ } else if (this.config.getDataType() == DataType.TOPIC_CONFIG.ordinal()) {
+ return pollTopicConfig();
+ } else if (this.config.getDataType() == DataType.BROKER_CONFIG.ordinal()) {
+ return pollBrokerConfig();
+ } else {
+ return pollSubConfig();
+ }
+ }
+
+ public void start(KeyValue config) {
+ ConfigUtil.load(config, this.config);
+ this.consumer.setConsumerGroup(Utils.createGroupName(this.config.getSourceTopic()));
+ this.consumer.setNamesrvAddr(this.config.getSourceRocketmq());
+ try {
+ this.consumer.start();
+ Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(this.config.getSourceTopic());
+ if (!this.config.getQueueId().equals("")) {
+ for (MessageQueue mq: mqs) {
+ if (Integer.valueOf(this.config.getQueueId()) == mq.getQueueId()) {
+ ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
+ ByteBuffer.wrap(RmqConstants.getPartition(mq.getTopic(),
+ mq.getBrokerName(), String.valueOf(mq.getQueueId()))
+ .getBytes("UTF-8")));
+
+ if (null != positionInfo && positionInfo.array().length > 0) {
+ String positionJson = new String(positionInfo.array(), "UTF-8");
+ JSONObject jsonObject = JSONObject.parseObject(positionJson);
+ this.config.setNextPosition(jsonObject.getLong(RmqConstants.NEXT_POSITION));
+ }
+ mqOffsetMap.put(mq, this.config.getNextPosition());
+ }
+ }
+ } else {
+ for (MessageQueue mq: mqs) {
+ ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
+ ByteBuffer.wrap(RmqConstants.getPartition(mq.getTopic(),
+ mq.getBrokerName(), String.valueOf(mq.getQueueId()))
+ .getBytes("UTF-8")));
+
+ if (null != positionInfo && positionInfo.array().length > 0) {
+ String positionJson = new String(positionInfo.array(), "UTF-8");
+ JSONObject jsonObject = JSONObject.parseObject(positionJson);
+ this.config.setNextPosition(jsonObject.getLong(RmqConstants.NEXT_POSITION));
+ }
+ mqOffsetMap.put(mq, this.config.getNextPosition());
+ }
+ }
+
+ } catch (Exception e) {
+ log.error("consumer of task {} start failed. ", this.taskId, e);
+ }
+ }
+
+ public void stop() {
+ this.consumer.shutdown();
+ }
+
+ public void pause() {
+
+ }
+
+ public void resume() {
+
+ }
+
+ private Collection<SourceDataEntry> pollCommonMessage() {
+
+ List<SourceDataEntry> res = new ArrayList<SourceDataEntry>();
+
+ try {
+ for (MessageQueue mq : this.mqOffsetMap.keySet()) {
+ PullResult pullResult = consumer.pull(mq, "*", this.mqOffsetMap.get(mq), 32);
+ switch (pullResult.getPullStatus()) {
+ case FOUND: {
+ this.mqOffsetMap.put(mq, pullResult.getNextBeginOffset());
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put(RmqConstants.NEXT_POSITION, pullResult.getNextBeginOffset());
+
+ List<MessageExt> msgs = pullResult.getMsgFoundList();
+ for (MessageExt m : msgs) {
+ Schema schema = new Schema();
+ schema.setDataSource(this.config.getSourceRocketmq());
+ schema.setName(this.config.getSourceTopic());
+ schema.setFields(new ArrayList<Field>());
+ schema.getFields().add(new Field(0, FieldName.COMMON_MESSAGE.getKey(), FieldType.STRING));
+
+ DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
+ dataEntryBuilder.timestamp(System.currentTimeMillis())
+ .queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
+ dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), JSONObject.toJSONString(m));
+ SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
+ ByteBuffer.wrap(RmqConstants.getPartition(this.config.getSourceTopic(),
+ this.config.getBrokerName(),
+ this.config.getQueueId()).getBytes("UTF-8")),
+ ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8"))
+ );
+ res.add(sourceDataEntry);
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ } catch (Exception e) {
+ log.error("Rocketmq connector task poll error, current config: {}", JSON.toJSONString(config), e);
+ }
+
+ return res;
+ }
+
+ private Collection<SourceDataEntry> pollTopicConfig() {
+ return new ArrayList<SourceDataEntry>();
+ }
+
+ private Collection<SourceDataEntry> pollBrokerConfig() {
+ return new ArrayList<SourceDataEntry>();
+ }
+
+ private Collection<SourceDataEntry> pollSubConfig() {
+ return new ArrayList<SourceDataEntry>();
+ }
+}
+
diff --git a/src/main/java/org/apache/rocketmq/connector/common/Utils.java b/src/main/java/org/apache/rocketmq/connector/common/Utils.java
new file mode 100644
index 0000000..71f538c
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/common/Utils.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.common;
+
+public class Utils {
+
+ public static String createGroupName(String prefix) {
+ return new StringBuilder().append(prefix).append("_").append(System.currentTimeMillis()).toString();
+ }
+
+ public static String createTaskId(String prefix) {
+ return new StringBuilder().append(prefix).append("_").append(System.currentTimeMillis()).toString();
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java b/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java
new file mode 100644
index 0000000..64e9c94
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.config;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class ConfigDefine {
+
+ public static final String SOURCE_RMQ = "sourceRocketmq";
+
+ public static final String STORE_TOPIC = "storeTopic";
+
+ public static final String TARGET_RMQ = "targetRocketmq";
+
+ public static final String DATA_TYPE = "dataType";
+
+ public static final String QUEUE_ID = "queueId";
+
+ public static final String TASK_DIVIDE_STRATEGY = "taskDivideStrategy";
+
+ public static final String BROKER_NAME = "brokerName";
+
+ public static final String SOURCE_TOPIC = "sourceTopic";
+
+ public static final Set<String> REQUEST_CONFIG = new HashSet<String>(){
+ {
+ add("sourceRocketmq");
+ add("targetRocketmq");
+ add("storeTopic");
+ add("taskDivideStrategy");
+ }
+ };
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/config/ConfigUtil.java b/src/main/java/org/apache/rocketmq/connector/config/ConfigUtil.java
new file mode 100644
index 0000000..d2dd7da
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/config/ConfigUtil.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.config;
+
+import io.openmessaging.KeyValue;
+
+import java.lang.reflect.Method;
+
+public class ConfigUtil {
+
+ public static <T> void load(KeyValue props, Object object) {
+
+ properties2Object(props, object);
+ }
+
+ private static <T> void properties2Object(final KeyValue p, final Object object) {
+
+ Method[] methods = object.getClass().getMethods();
+ for (Method method : methods) {
+ String mn = method.getName();
+ if (mn.startsWith("set")) {
+ try {
+ String tmp = mn.substring(4);
+ String first = mn.substring(3, 4);
+
+ String key = first.toLowerCase() + tmp;
+ String property = p.getString(key);
+ if (property != null) {
+ Class<?>[] pt = method.getParameterTypes();
+ if (pt != null && pt.length > 0) {
+ String cn = pt[0].getSimpleName();
+ Object arg;
+ if (cn.equals("int") || cn.equals("Integer")) {
+ arg = Integer.parseInt(property);
+ } else if (cn.equals("long") || cn.equals("Long")) {
+ arg = Long.parseLong(property);
+ } else if (cn.equals("double") || cn.equals("Double")) {
+ arg = Double.parseDouble(property);
+ } else if (cn.equals("boolean") || cn.equals("Boolean")) {
+ arg = Boolean.parseBoolean(property);
+ } else if (cn.equals("float") || cn.equals("Float")) {
+ arg = Float.parseFloat(property);
+ } else if (cn.equals("String")) {
+ arg = property;
+ } else {
+ continue;
+ }
+ method.invoke(object, arg);
+ }
+ }
+ } catch (Throwable ignored) {
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/config/DataType.java b/src/main/java/org/apache/rocketmq/connector/config/DataType.java
new file mode 100644
index 0000000..3e77a3a
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/config/DataType.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.config;
+
+public enum DataType {
+
+ COMMON_MESSAGE,
+ TOPIC_CONFIG,
+ BROKER_CONFIG,
+ SUB_CONFIG
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java b/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
new file mode 100644
index 0000000..7bba3fb
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.config;
+
+public class TaskConfig {
+
+ private String storeTopic;
+ private String sourceTopic;
+ private String sourceRocketmq;
+ private Integer dataType;
+ private String brokerName;
+ private String queueId;
+ private Long nextPosition;
+
+ public String getStoreTopic() {
+ return storeTopic;
+ }
+
+ public void setStoreTopic(String storeTopic) {
+ this.storeTopic = storeTopic;
+ }
+
+ public String getSourceTopic() {
+ return sourceTopic;
+ }
+
+ public void setSourceTopic(String sourceTopic) {
+ this.sourceTopic = sourceTopic;
+ }
+
+ public String getSourceRocketmq() {
+ return sourceRocketmq;
+ }
+
+ public void setSourceRocketmq(String sourceRocketmq) {
+ this.sourceRocketmq = sourceRocketmq;
+ }
+
+ public int getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(int dataType) {
+ this.dataType = dataType;
+ }
+
+ public void setDataType(Integer dataType) {
+ this.dataType = dataType;
+ }
+
+ public String getBrokerName() {
+ return brokerName;
+ }
+
+ public void setBrokerName(String brokerName) {
+ this.brokerName = brokerName;
+ }
+
+ public String getQueueId() {
+ return queueId;
+ }
+
+ public void setQueueId(String queueId) {
+ this.queueId = queueId;
+ }
+
+ public Long getNextPosition() {
+ return nextPosition;
+ }
+
+ public void setNextPosition(Long nextPosition) {
+ this.nextPosition = nextPosition;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/schema/FieldName.java b/src/main/java/org/apache/rocketmq/connector/schema/FieldName.java
new file mode 100644
index 0000000..9f4dc47
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/schema/FieldName.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.schema;
+
+public enum FieldName {
+ COMMON_MESSAGE("MessageExt");
+
+ private String key;
+
+ FieldName(String key) {
+ this.key = key;
+ }
+
+ public String getKey() {
+ return key;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideStrategyEnum.java b/src/main/java/org/apache/rocketmq/connector/strategy/DivideStrategyEnum.java
new file mode 100644
index 0000000..9dc060f
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/strategy/DivideStrategyEnum.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.strategy;
+
+public enum DivideStrategyEnum {
+
+ BY_TOPIC,
+ BY_QUEUE
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java
new file mode 100644
index 0000000..b6eae8f
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.strategy;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.internal.DefaultKeyValue;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.connector.config.ConfigDefine;
+import org.apache.rocketmq.connector.config.DataType;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class DivideTaskByQueue extends TaskDivideStrategy {
+ public List<KeyValue> divide(Map<String, List<MessageQueue>> topicRouteMap, String source, String storeTopic) {
+
+ List<KeyValue> config = new ArrayList<KeyValue>();
+
+ for (String t: topicRouteMap.keySet()) {
+ for (MessageQueue mq: topicRouteMap.get(t)) {
+ KeyValue keyValue = new DefaultKeyValue();
+ keyValue.put(ConfigDefine.STORE_TOPIC, storeTopic);
+ keyValue.put(ConfigDefine.SOURCE_RMQ, source);
+ keyValue.put(ConfigDefine.STORE_TOPIC, t);
+ keyValue.put(ConfigDefine.BROKER_NAME, mq.getBrokerName());
+ keyValue.put(ConfigDefine.QUEUE_ID, String.valueOf(mq.getQueueId()));
+ keyValue.put(ConfigDefine.SOURCE_TOPIC, t);
+ keyValue.put(ConfigDefine.DATA_TYPE, DataType.COMMON_MESSAGE.ordinal());
+ config.add(keyValue);
+ }
+ }
+
+ return config;
+
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
new file mode 100644
index 0000000..7c96b28
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.strategy;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.internal.DefaultKeyValue;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.connector.config.ConfigDefine;
+import org.apache.rocketmq.connector.config.DataType;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class DivideTaskByTopic extends TaskDivideStrategy {
+
+ public List<KeyValue> divide(Map<String, List<MessageQueue>> topicRouteMap, String source, String storeTopic) {
+
+ List<KeyValue> config = new ArrayList<KeyValue>();
+
+ for (String t: topicRouteMap.keySet()) {
+ KeyValue keyValue = new DefaultKeyValue();
+ keyValue.put(ConfigDefine.STORE_TOPIC, storeTopic);
+ keyValue.put(ConfigDefine.SOURCE_RMQ, source);
+ keyValue.put(ConfigDefine.STORE_TOPIC, t);
+ keyValue.put(ConfigDefine.BROKER_NAME, "");
+ keyValue.put(ConfigDefine.QUEUE_ID, "");
+ keyValue.put(ConfigDefine.SOURCE_TOPIC, t);
+ keyValue.put(ConfigDefine.DATA_TYPE, DataType.COMMON_MESSAGE.ordinal());
+ config.add(keyValue);
+ }
+
+ return config;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java b/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java
new file mode 100644
index 0000000..f847cb1
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connector.strategy;
+
+import io.openmessaging.KeyValue;
+import org.apache.rocketmq.common.message.MessageQueue;
+
+import java.util.List;
+import java.util.Map;
+
+public abstract class TaskDivideStrategy {
+
+ public abstract List<KeyValue> divide(Map<String, List<MessageQueue>> topicMap, String source, String storeTopic);
+}
[rocketmq-connect] 11/39: Topic rename. resolve #399
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 81376b9a25448472d1afd2b45088882a8851187f
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Mon Sep 9 11:31:23 2019 +0800
Topic rename. resolve #399
---
.../rocketmq/replicator/RmqSourceReplicator.java | 27 ++++++++---
.../apache/rocketmq/replicator/RmqSourceTask.java | 14 +++---
.../apache/rocketmq/replicator/common/Utils.java | 6 +--
.../rocketmq/replicator/config/ConfigDefine.java | 4 +-
.../rocketmq/replicator/config/TaskTopicInfo.java | 19 +++++++-
.../replicator/strategy/DivideTaskByQueue.java | 7 +--
.../replicator/strategy/DivideTaskByTopic.java | 10 ++--
.../replicator/strategy/TaskDivideStrategy.java | 3 +-
.../replicator/RmqSourceReplicatorTest.java | 53 +++++-----------------
9 files changed, 74 insertions(+), 69 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index 6ea3ae8..f24cf63 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -27,6 +27,8 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.text.StrSubstitutor;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
@@ -42,6 +44,7 @@ import org.apache.rocketmq.replicator.common.Utils;
import org.apache.rocketmq.replicator.config.ConfigDefine;
import org.apache.rocketmq.replicator.config.DataType;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
+import org.apache.rocketmq.replicator.config.TaskTopicInfo;
import org.apache.rocketmq.replicator.strategy.DivideStrategyEnum;
import org.apache.rocketmq.replicator.strategy.DivideTaskByQueue;
import org.apache.rocketmq.replicator.strategy.DivideTaskByTopic;
@@ -60,7 +63,7 @@ public class RmqSourceReplicator extends SourceConnector {
private KeyValue replicatorConfig;
- private Map<String, List<MessageQueue>> topicRouteMap;
+ private Map<String, List<TaskTopicInfo>> topicRouteMap;
private TaskDivideStrategy taskDivideStrategy;
@@ -76,7 +79,7 @@ public class RmqSourceReplicator extends SourceConnector {
private volatile boolean adminStarted;
public RmqSourceReplicator() {
- topicRouteMap = new HashMap<String, List<MessageQueue>>();
+ topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
whiteList = new HashSet<String>();
}
@@ -204,8 +207,9 @@ public class RmqSourceReplicator extends SourceConnector {
for (Pattern pattern : patterns) {
Matcher matcher = pattern.matcher(topic);
if (matcher.matches()) {
+ String targetTopic = generateTargetTopic(topic);
if (!targetTopicSet.contains(topic)) {
- ensureTargetTopic(topic, topic);
+ ensureTargetTopic(topic, targetTopic);
}
// different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
@@ -219,13 +223,13 @@ public class RmqSourceReplicator extends SourceConnector {
TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
if (!topicRouteMap.containsKey(topic)) {
- topicRouteMap.put(topic, new ArrayList<MessageQueue>());
+ topicRouteMap.put(topic, new ArrayList<TaskTopicInfo>());
}
for (QueueData qd : topicRouteData.getQueueDatas()) {
if (brokerNameSet.contains(qd.getBrokerName())) {
for (int i = 0; i < qd.getReadQueueNums(); i++) {
- MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
- topicRouteMap.get(topic).add(mq);
+ TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), String.valueOf(i), targetTopic);
+ topicRouteMap.get(topic).add(taskTopicInfo);
}
}
}
@@ -244,7 +248,7 @@ public class RmqSourceReplicator extends SourceConnector {
this.whiteList = whiteList;
}
- public Map<String, List<MessageQueue>> getTopicRouteMap() {
+ public Map<String, List<TaskTopicInfo>> getTopicRouteMap() {
return this.topicRouteMap;
}
@@ -283,5 +287,14 @@ public class RmqSourceReplicator extends SourceConnector {
throw new IllegalStateException("");
}
+ public String generateTargetTopic(String topic) {
+ String fmt = this.replicatorConfig.getString(ConfigDefine.CONN_TOPIC_RENAME_FMT);
+ if (StringUtils.isNotEmpty(fmt)) {
+ Map<String, String> params = new HashMap<String, String>();
+ params.put("topic", topic);
+ return StrSubstitutor.replace(fmt, params);
+ }
+ return topic;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index 9909d61..c4d53fa 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -46,12 +46,12 @@ public class RmqSourceTask extends SourceTask {
private final DefaultMQPullConsumer consumer;
private volatile boolean started = false;
- private Map<MessageQueue, Long> mqOffsetMap;
+ private Map<TaskTopicInfo, Long> mqOffsetMap;
public RmqSourceTask() {
this.config = new TaskConfig();
this.consumer = new DefaultMQPullConsumer();
this.taskId = Utils.createTaskId(Thread.currentThread().getName());
- mqOffsetMap = new HashMap<MessageQueue, Long>();
+ mqOffsetMap = new HashMap<TaskTopicInfo, Long>();
}
public Collection<SourceDataEntry> poll() {
@@ -95,7 +95,7 @@ public class RmqSourceTask extends SourceTask {
} else {
this.config.setNextPosition(0L);
}
- mqOffsetMap.put(mq, this.config.getNextPosition());
+ mqOffsetMap.put(tti, this.config.getNextPosition());
}
}
} else {
@@ -114,7 +114,7 @@ public class RmqSourceTask extends SourceTask {
} else {
this.config.setNextPosition(0L);
}
- mqOffsetMap.put(mq, this.config.getNextPosition());
+ mqOffsetMap.put(tti, this.config.getNextPosition());
}
}
}
@@ -148,12 +148,13 @@ public class RmqSourceTask extends SourceTask {
List<SourceDataEntry> res = new ArrayList<SourceDataEntry>();
if (started) {
try {
- for (MessageQueue mq : this.mqOffsetMap.keySet()) {
+ for (TaskTopicInfo taskTopicConfig : this.mqOffsetMap.keySet()) {
+ MessageQueue mq = taskTopicConfig.convertMQ();
PullResult pullResult = consumer.pull(mq, "*",
this.mqOffsetMap.get(mq), 32);
switch (pullResult.getPullStatus()) {
case FOUND: {
- this.mqOffsetMap.put(mq, pullResult.getNextBeginOffset());
+ this.mqOffsetMap.put(taskTopicConfig, pullResult.getNextBeginOffset());
JSONObject jsonObject = new JSONObject();
jsonObject.put(RmqConstants.NEXT_POSITION, pullResult.getNextBeginOffset());
List<MessageExt> msgs = pullResult.getMsgFoundList();
@@ -175,6 +176,7 @@ public class RmqSourceTask extends SourceTask {
String.valueOf(mq.getQueueId())).getBytes("UTF-8")),
ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8"))
);
+ sourceDataEntry.setQueueName(taskTopicConfig.getTargetTopic());
res.add(sourceDataEntry);
break;
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index e5c0866..f3e6be6 100644
--- a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
@@ -58,12 +58,12 @@ public class Utils {
return String.valueOf(namesrvList.toString().hashCode());
}
-
- public static List<BrokerData> examineBrokerData(DefaultMQAdminExt defaultMQAdminExt, String topic, String cluster) throws RemotingException, MQClientException, InterruptedException {
+ public static List<BrokerData> examineBrokerData(DefaultMQAdminExt defaultMQAdminExt, String topic,
+ String cluster) throws RemotingException, MQClientException, InterruptedException {
List<BrokerData> brokerList = new ArrayList<BrokerData>();
TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
- if (topicRouteData.getBrokerDatas() != null) { // check下
+ if (topicRouteData.getBrokerDatas() != null) {
for (BrokerData broker : topicRouteData.getBrokerDatas()) {
if (StringUtils.equals(broker.getCluster(), cluster)) {
brokerList.add(broker);
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
index ddf4972..5b15821 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
@@ -41,10 +41,12 @@ public class ConfigDefine {
public static final String CONN_TASK_PARALLELISM = "task-parallelism";
+ public static final String CONN_TOPIC_RENAME_FMT = "topic.rename.format";
+
/**
* The required key for all configurations.
*/
- public static final Set<String> REQUEST_CONFIG = new HashSet<String>(){
+ public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
{
add(CONN_SOURCE_RMQ);
add(CONN_TARGET_RMQ);
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
index f078028..c5a39e4 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
@@ -16,16 +16,20 @@
*/
package org.apache.rocketmq.replicator.config;
+import org.apache.rocketmq.common.message.MessageQueue;
+
public class TaskTopicInfo {
private String sourceTopic;
private String brokerName;
private String queueId;
+ private String targetTopic;
- public TaskTopicInfo(String sourceTopic, String brokerName, String queueId) {
+ public TaskTopicInfo(String sourceTopic, String brokerName, String queueId, String targetTopic) {
this.sourceTopic = sourceTopic;
this.brokerName = brokerName;
this.queueId = queueId;
+ this.targetTopic = targetTopic;
}
public String getSourceTopic() {
@@ -51,4 +55,17 @@ public class TaskTopicInfo {
public void setQueueId(String queueId) {
this.queueId = queueId;
}
+
+ public String getTargetTopic() {
+ return this.targetTopic;
+ }
+
+ public void setTargetTopic(String targetTopic) {
+ this.targetTopic = targetTopic;
+ }
+
+ public MessageQueue convertMQ() {
+ return new MessageQueue(sourceTopic,
+ brokerName, Integer.parseInt(queueId));
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
index 77ed871..d6a15ad 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
@@ -22,14 +22,15 @@ import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.rocketmq.replicator.config.TaskTopicInfo;
public class DivideTaskByQueue extends TaskDivideStrategy {
- public List<KeyValue> divide(Map<String, List<MessageQueue>> topicRouteMap, TaskDivideConfig tdc) {
+ public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
- for (String t: topicRouteMap.keySet()) {
- for (MessageQueue mq: topicRouteMap.get(t)) {
+ for (String t : topicRouteMap.keySet()) {
+ for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
}
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
index e667d57..f8dc8cf 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
@@ -28,21 +28,21 @@ import java.util.Map;
public class DivideTaskByTopic extends TaskDivideStrategy {
- public List<KeyValue> divide(Map<String, List<MessageQueue>> topicRouteMap, TaskDivideConfig tdc) {
+ public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
int id = -1;
Map<Integer, List<TaskTopicInfo>> taskTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
- for (String t: topicRouteMap.keySet()) {
- int ind = ++id%parallelism;
+ for (Map.Entry<String, List<TaskTopicInfo>> entry : topicRouteMap.entrySet()) {
+ int ind = ++id % parallelism;
if (!taskTopicList.containsKey(ind)) {
taskTopicList.put(ind, new ArrayList<TaskTopicInfo>());
}
- taskTopicList.get(ind).add(new TaskTopicInfo(t, "", ""));
+ taskTopicList.get(ind).addAll(entry.getValue());
}
- for (int i=0; i<parallelism; i++) {
+ for (int i = 0; i < parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java b/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
index 0e0ac99..093d92e 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
@@ -21,8 +21,9 @@ import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import java.util.List;
import java.util.Map;
+import org.apache.rocketmq.replicator.config.TaskTopicInfo;
public abstract class TaskDivideStrategy {
- public abstract List<KeyValue> divide(Map<String, List<MessageQueue>> topicMap, TaskDivideConfig tdc);
+ public abstract List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicMap, TaskDivideConfig tdc);
}
diff --git a/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java b/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
index 6ac0049..f271f14 100644
--- a/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
+++ b/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
@@ -29,7 +29,6 @@ import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
-import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
@@ -42,51 +41,21 @@ import org.mockito.Mockito;
import org.mockito.internal.util.reflection.FieldSetter;
import org.mockito.junit.MockitoJUnitRunner;
+import io.openmessaging.KeyValue;
+import io.openmessaging.internal.DefaultKeyValue;
+import org.apache.rocketmq.replicator.config.ConfigDefine;
+
@RunWith(MockitoJUnitRunner.class)
public class RmqSourceReplicatorTest {
- @Mock
- private DefaultMQAdminExt defaultMQAdminExt;
-
-
@Test
- public void buildWildcardRoute() throws RemotingException, MQClientException, InterruptedException, NoSuchFieldException {
-
+ public void testGenerateTopic() throws NoSuchFieldException {
RmqSourceReplicator rmqSourceReplicator = Mockito.spy(RmqSourceReplicator.class);
-
- TopicList topicList = new TopicList();
- Set<String> topics = new HashSet<String>();
- topics.add("topic1");
- topics.add("topic2");
- topics.add("sub-topic1-test");
- topics.add("sub-topic2-test");
- topics.add("sub-topic2-xxx");
- topics.add("sub-0");
- topics.add("test-0");
- topics.add("0-test");
- topicList.setTopicList(topics);
- when(defaultMQAdminExt.fetchAllTopicList()).thenReturn(topicList);
-
- TopicRouteData topicRouteData = new TopicRouteData();
- topicRouteData.setQueueDatas(Collections.<QueueData>emptyList());
- when(defaultMQAdminExt.examineTopicRouteInfo(any(String.class))).thenReturn(topicRouteData);
-
-
- Field field = RmqSourceReplicator.class.getDeclaredField("defaultMQAdminExt");
- FieldSetter.setField(rmqSourceReplicator, field, defaultMQAdminExt);
-
- Set<String> whiteList = new HashSet<String>();
- whiteList.add("topic1");
- whiteList.add("\\w+-test");
- rmqSourceReplicator.setWhiteList(whiteList);
- rmqSourceReplicator.buildRoute();
- Map<String, List<MessageQueue>> queues = rmqSourceReplicator.getTopicRouteMap();
- Set<String> expected = new HashSet<String>();
- expected.add("topic1");
- expected.add("0-test");
- assertThat(queues.size()).isEqualTo(expected.size());
- for (String topic : expected) {
- assertThat(queues.containsKey(topic)).isTrue();
- }
+ KeyValue kv = new DefaultKeyValue();
+ kv.put(ConfigDefine.CONN_TOPIC_RENAME_FMT, "${topic}.replica");
+ Field field = RmqSourceReplicator.class.getDeclaredField("replicatorConfig");
+ FieldSetter.setField(rmqSourceReplicator, field, kv);
+ String dstTopic = rmqSourceReplicator.generateTargetTopic("dest");
+ assertThat(dstTopic).isEqualTo("dest.replica");
}
}
[rocketmq-connect] 36/39: [rocketmq-replicator] Bug fix: Missing subscription group when syncing offset (#837)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 691ff01d1e08d5432ef39634478ebc1aab058904
Author: Git_Yang <30...@users.noreply.github.com>
AuthorDate: Wed Oct 27 14:13:12 2021 +0800
[rocketmq-replicator] Bug fix: Missing subscription group when syncing offset (#837)
Signed-off-by: zhangyang21 <zh...@xiaomi.com>
---
.../apache/rocketmq/replicator/MetaSourceTask.java | 2 +-
.../rocketmq/replicator/offset/OffsetSync.java | 24 +++++++++++++++-------
.../replicator/offset/OffsetSyncStore.java | 10 ++++-----
3 files changed, 23 insertions(+), 13 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
index a66ba22..67fc89f 100644
--- a/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
@@ -120,7 +120,7 @@ public class MetaSourceTask extends SourceTask {
MessageQueue mq = offsetTable.getKey();
long srcOffset = offsetTable.getValue().getConsumerOffset();
- long targetOffset = this.store.convertTargetOffset(mq, srcOffset);
+ long targetOffset = this.store.convertTargetOffset(mq, group, srcOffset);
JSONObject jsonObject = new JSONObject();
jsonObject.put(RmqConstants.NEXT_POSITION, srcOffset);
diff --git a/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java b/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
index 4678d5b..51c29a7 100644
--- a/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
+++ b/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
@@ -21,13 +21,15 @@ import org.apache.rocketmq.common.message.MessageQueue;
public class OffsetSync {
private MessageQueue mq;
+ private String group;
private long srcOffset;
- private long targtOffset;
+ private long targetOffset;
- public OffsetSync(MessageQueue mq, long srcOffset, long targtOffset) {
+ public OffsetSync(MessageQueue mq, String group, long srcOffset, long targetOffset) {
this.mq = mq;
+ this.group = group;
this.srcOffset = srcOffset;
- this.targtOffset = targtOffset;
+ this.targetOffset = targetOffset;
}
public void setMq(MessageQueue mq) {
@@ -38,22 +40,30 @@ public class OffsetSync {
this.srcOffset = srcOffset;
}
- public void setTargtOffset(long targtOffset) {
- this.targtOffset = targtOffset;
+ public void setTargetOffset(long targetOffset) {
+ this.targetOffset = targetOffset;
}
public long getSrcOffset() {
return this.srcOffset;
}
- public long getTargtOffset() {
- return this.targtOffset;
+ public long getTargetOffset() {
+ return this.targetOffset;
}
public MessageQueue getMq() {
return this.mq;
}
+ public String getGroup() {
+ return group;
+ }
+
+ public void setGroup(String group) {
+ this.group = group;
+ }
+
public byte[] encode() {
return JSON.toJSONBytes(this);
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java b/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
index 5703f39..f8c4b6a 100644
--- a/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
+++ b/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
@@ -48,13 +48,13 @@ public class OffsetSyncStore {
this.consumer = new DefaultMQPullConsumer();
}
- public long convertTargetOffset(MessageQueue mq, long srcOffset) {
- OffsetSync offsetSync = latestOffsetSync(mq);
+ public long convertTargetOffset(MessageQueue mq, String group, long srcOffset) {
+ OffsetSync offsetSync = latestOffsetSync(mq, group);
if (offsetSync.getSrcOffset() > srcOffset) {
return -1;
}
long delta = srcOffset - offsetSync.getSrcOffset();
- return offsetSync.getTargtOffset() + delta;
+ return offsetSync.getTargetOffset() + delta;
}
private boolean sync(
@@ -79,10 +79,10 @@ public class OffsetSyncStore {
}
}
- private OffsetSync latestOffsetSync(MessageQueue queue) {
+ private OffsetSync latestOffsetSync(MessageQueue queue, String group) {
return syncs.computeIfAbsent(queue, new Function<MessageQueue, OffsetSync>() {
@Override public OffsetSync apply(MessageQueue queue) {
- return new OffsetSync(queue, -1, -1);
+ return new OffsetSync(queue, group, -1, -1);
}
});
}
[rocketmq-connect] 25/39: The topic name in the TaskTopicInfo constructor should be topic (#473)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 5ed354b5015a7a1d42f263e5f575b5c5699c3cc3
Author: zhoubo <87...@qq.com>
AuthorDate: Fri Nov 29 21:30:26 2019 +0800
The topic name in the TaskTopicInfo constructor should be topic (#473)
https://github.com/apache/rocketmq-externals/issues/470
---
.../java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
index 3e2962b..b791b93 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
@@ -22,8 +22,8 @@ public class TaskTopicInfo extends MessageQueue {
private String targetTopic;
- public TaskTopicInfo(String sourceTopic, String brokerName, int queueId, String targetTopic) {
- super(sourceTopic, brokerName, queueId);
+ public TaskTopicInfo(String topic, String brokerName, int queueId, String targetTopic) {
+ super(topic, brokerName, queueId);
this.targetTopic = targetTopic;
}
[rocketmq-connect] 05/39: return taskConfig with source-record-converter (#373)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 25e699e49745b43bb0c8c07bae29c630cdf73b63
Author: mike_xwm <mi...@126.com>
AuthorDate: Fri Aug 9 21:17:17 2019 +0800
return taskConfig with source-record-converter (#373)
---
src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java | 3 ++-
.../java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java b/src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java
index d8aa08a..fca4dcc 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java
+++ b/src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java
@@ -27,7 +27,8 @@ public enum TaskConfigEnum {
TASK_BROKER_NAME("brokerName"),
TASK_QUEUE_ID("queueId"),
TASK_NEXT_POSITION("nextPosition"),
- TASK_TOPIC_INFO("taskTopicList");
+ TASK_TOPIC_INFO("taskTopicList"),
+ TASK_SOURCE_RECORD_CONVERTER("source-record-converter");
private String key;
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
index 53699a9..9b019f1 100644
--- a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
+++ b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
@@ -49,6 +49,7 @@ public class DivideTaskByTopic extends TaskDivideStrategy {
keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(taskTopicList.get(i)));
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
config.add(keyValue);
}
[rocketmq-connect] 01/39: (1)Rename rocketmq-connector to replicator (2) Initialize RocketMQ activemq connect and runtime
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 952aa8fdecf0dd6ec2be203ab5fadb7469d12b33
Author: duheng.dh <du...@alibaba-inc.com>
AuthorDate: Fri May 31 19:08:13 2019 +0800
(1)Rename rocketmq-connector to replicator (2) Initialize RocketMQ activemq connect and runtime
---
pom.xml | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..0c79eee
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-connector</artifactId>
+ <version>1.0-SNAPSHOT</version>
+
+
+</project>
\ No newline at end of file
[rocketmq-connect] 03/39: Define and Implement the RmqConnector and RmqSourceTask. (#343)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 6ddd7e447b529185970c388541399d4b5a576bb2
Author: chuenfaiy <ch...@163.com>
AuthorDate: Fri Jul 26 21:29:10 2019 +0800
Define and Implement the RmqConnector and RmqSourceTask. (#343)
* [Fix]fix some bugs in connectors and fastjson.
* [Update]update the version of replicator
* [Update]remove the repetitive properties
* [Update]Add the license at the head of TaskTopicInfo class and TaskConfigEnum class
---
.gitignore | 1 -
README.md | 13 ++
pom.xml | 12 +-
.../rocketmq/connector/RmqSourceConnector.java | 131 +++++++++++++++------
.../apache/rocketmq/connector/RmqSourceTask.java | 120 +++++++++++--------
.../common/{Utils.java => ConstDefine.java} | 10 +-
.../apache/rocketmq/connector/common/Utils.java | 8 +-
.../rocketmq/connector/config/ConfigDefine.java | 34 ++++--
.../rocketmq/connector/config/TaskConfig.java | 45 +++----
.../Utils.java => config/TaskConfigEnum.java} | 25 +++-
.../{TaskConfig.java => TaskDivideConfig.java} | 75 ++++++------
.../config/{TaskConfig.java => TaskTopicInfo.java} | 44 +------
.../connector/strategy/DivideTaskByQueue.java | 12 +-
.../connector/strategy/DivideTaskByTopic.java | 30 +++--
.../connector/strategy/TaskDivideStrategy.java | 3 +-
15 files changed, 307 insertions(+), 256 deletions(-)
diff --git a/.gitignore b/.gitignore
index 36716aa..525eaaa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,3 @@
-*.iml
*.class
*.jar
*dependency-reduced-pom.xml
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..0b8bef7
--- /dev/null
+++ b/README.md
@@ -0,0 +1,13 @@
+# rocketmq-replicator
+
+# 启动参数选择
+
+参数 | 类型 |是否必须 |说明|示例值
+---|---|---|---|---|
+source-rocketmq | 字符串 | 是 | 源rocketmq集群namesrv地址 | 192.168.1.2:9876 |
+target-rocketmq | 字符串 | 是 | 源rocketmq集群namesrv地址 | 192.168.1.2:9876 |
+replicator-store-topic | 字符串 | 是 | replicator存储topic，需要在runtime的mq集群提前创建 | replicator-store-topic |
+task-divide-strategy | 整型 | 否 | 任务切割策略，可以按照主题和队列来切割，目前只支持主题切割且主题对应值为0 | 0 |
+white-list | 字符串 | 是 | 复制主题白名单，多个topic之间使用逗号分隔 | topic-1,topic-2 |
+task-parallelism | 整型 | 否 | 任务并行度，默认值为1，当topic数大于task数时，一个task将负责多个topic | 2 |
+source-record-converter | 字符串 | 是 | 源数据解析器，目前使用的是Json解析器 | io.openmessaging.connect.runtime.converter.JsonConverter |
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 13a5af4..37bf78f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,7 +23,7 @@
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-connector</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>0.0.1-SNAPSHOT</version>
<properties>
<rocketmq.version>4.4.0</rocketmq.version>
@@ -72,16 +72,6 @@
<version>0.9.6</version>
</dependency>
<dependency>
- <groupId>io.openmessaging</groupId>
- <artifactId>openmessaging-connect</artifactId>
- <version>0.1.0-beta</version>
- </dependency>
- <dependency>
- <groupId>io.openmessaging</groupId>
- <artifactId>openmessaging-connector</artifactId>
- <version>0.1.0-beta</version>
- </dependency>
- <dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.51</version>
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java b/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
index c919fa3..2b5bda4 100644
--- a/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
@@ -19,13 +19,16 @@ package org.apache.rocketmq.connector;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.source.SourceConnector;
+import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.constant.PermName;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.common.protocol.body.TopicList;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.connector.common.ConstDefine;
+import org.apache.rocketmq.connector.common.Utils;
import org.apache.rocketmq.connector.config.ConfigDefine;
+import org.apache.rocketmq.connector.config.DataType;
+import org.apache.rocketmq.connector.config.TaskDivideConfig;
import org.apache.rocketmq.connector.strategy.DivideStrategyEnum;
import org.apache.rocketmq.connector.strategy.DivideTaskByQueue;
import org.apache.rocketmq.connector.strategy.DivideTaskByTopic;
@@ -35,48 +38,96 @@ import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
public class RmqSourceConnector extends SourceConnector {
private static final Logger log = LoggerFactory.getLogger(RmqSourceConnector.class);
+ private boolean syncDLQ = false;
+
+ private boolean syncRETRY = false;
+
private KeyValue config;
private Map<String, List<MessageQueue>> topicRouteMap;
- private final TaskDivideStrategy taskDivideStrategy;
+ private TaskDivideStrategy taskDivideStrategy;
+
+ private Set<String> whiteList;
+
+ private volatile boolean started = false;
+
+ private volatile boolean configValid = false;
+
+ private DefaultMQAdminExt defaultMQAdminExt;
+
+ private int taskParallelism = 1;
public RmqSourceConnector() {
topicRouteMap = new HashMap<String, List<MessageQueue>>();
-
- if (this.config.getInt(ConfigDefine.TASK_DIVIDE_STRATEGY) == DivideStrategyEnum.BY_TOPIC.ordinal()) {
- taskDivideStrategy = new DivideTaskByTopic();
- } else {
- taskDivideStrategy = new DivideTaskByQueue();
- }
+ whiteList = new HashSet<String>();
}
public String verifyAndSetConfig(KeyValue config) {
+ // check the need key.
for(String requestKey : ConfigDefine.REQUEST_CONFIG){
if(!config.containsKey(requestKey)){
return "Request config key: " + requestKey;
}
}
+
+ // check the whitelist, whitelist is required.
+ String whileListStr = this.config.getString(ConfigDefine.CONN_WHITE_LIST);
+ String[] wl = whileListStr.trim().split(",");
+ if (wl.length <= 0) return "White list must be not empty.";
+ else {
+ for (String t: wl) {
+ this.whiteList.add(t.trim());
+ }
+ }
+
+ if (this.config.containsKey(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) &&
+ this.config.getInt(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) == DivideStrategyEnum.BY_QUEUE.ordinal()) {
+ this.taskDivideStrategy = new DivideTaskByQueue();
+ } else {
+ this.taskDivideStrategy = new DivideTaskByTopic();
+ }
+
+ if (config.containsKey(ConfigDefine.CONN_TASK_PARALLELISM)) {
+ this.taskParallelism = this.config.getInt(ConfigDefine.CONN_TASK_PARALLELISM);
+ }
+
this.config = config;
+ this.configValid = true;
return "";
}
public void start() {
+
+ if (configValid) {
+ RPCHook rpcHook = null;
+ this.defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ this.defaultMQAdminExt.setNamesrvAddr(this.config.getString(ConfigDefine.CONN_SOURCE_RMQ));
+ this.defaultMQAdminExt.setInstanceName(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ try {
+ defaultMQAdminExt.start();
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `defaultMQAdminExt` exception.", e);
+ }
+ started = true;
+ }
}
public void stop() {
-
+ if (started) {
+ if (defaultMQAdminExt != null) {
+ defaultMQAdminExt.shutdown();
+ }
+ started = false;
+ }
}
public void pause() {
@@ -88,26 +139,24 @@ public class RmqSourceConnector extends SourceConnector {
}
public Class<? extends Task> taskClass() {
- return null;
+
+ return RmqSourceTask.class;
}
public List<KeyValue> taskConfigs() {
-
- System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, this.config.getString(ConfigDefine.SOURCE_RMQ));
- RPCHook rpcHook = null;
- DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
- defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis()));
- try {
- defaultMQAdminExt.start();
- TopicList topicList = defaultMQAdminExt.fetchAllTopicList();
- for (String topic: topicList.getTopicList()) {
- if (!topic.equals(ConfigDefine.STORE_TOPIC)) {
- TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
- if (!topicRouteMap.containsKey(topic)) {
- topicRouteMap.put(topic, new ArrayList<MessageQueue>());
- }
- for (QueueData qd: topicRouteData.getQueueDatas()) {
- if (PermName.isReadable(qd.getPerm())) {
+
+ if (started && configValid) {
+ try {
+ for (String topic : this.whiteList) {
+ if ((syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) ||
+ (syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) ||
+ !topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
+
+ TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+ if (!topicRouteMap.containsKey(topic)) {
+ topicRouteMap.put(topic, new ArrayList<MessageQueue>());
+ }
+ for (QueueData qd : topicRouteData.getQueueDatas()) {
for (int i = 0; i < qd.getReadQueueNums(); i++) {
MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
topicRouteMap.get(topic).add(mq);
@@ -115,15 +164,21 @@ public class RmqSourceConnector extends SourceConnector {
}
}
}
+ } catch (Exception e) {
+ log.error("Fetch topic list error.", e);
}
- } catch (Exception e) {
- log.error("fetch topic list error: ", e);
- } finally {
- defaultMQAdminExt.shutdown();
- }
- return this.taskDivideStrategy.divide(this.topicRouteMap,
- this.config.getString(ConfigDefine.SOURCE_RMQ), this.config.getString(ConfigDefine.STORE_TOPIC));
+ TaskDivideConfig tdc = new TaskDivideConfig(
+ this.config.getString(ConfigDefine.CONN_SOURCE_RMQ),
+ this.config.getString(ConfigDefine.CONN_STORE_TOPIC),
+ this.config.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER),
+ DataType.COMMON_MESSAGE.ordinal(),
+ this.taskParallelism
+ );
+ return this.taskDivideStrategy.divide(this.topicRouteMap, tdc);
+ } else {
+ return new ArrayList<KeyValue>();
+ }
}
}
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
index d71dc87..4fca698 100644
--- a/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
@@ -29,6 +29,7 @@ import org.apache.rocketmq.connector.common.Utils;
import org.apache.rocketmq.connector.config.ConfigUtil;
import org.apache.rocketmq.connector.config.DataType;
import org.apache.rocketmq.connector.config.TaskConfig;
+import org.apache.rocketmq.connector.config.TaskTopicInfo;
import org.apache.rocketmq.connector.schema.FieldName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,6 +44,7 @@ public class RmqSourceTask extends SourceTask {
private final String taskId;
private final TaskConfig config;
private final DefaultMQPullConsumer consumer;
+ private volatile boolean started = false;
private Map<MessageQueue, Long> mqOffsetMap;
public RmqSourceTask() {
@@ -67,50 +69,68 @@ public class RmqSourceTask extends SourceTask {
public void start(KeyValue config) {
ConfigUtil.load(config, this.config);
- this.consumer.setConsumerGroup(Utils.createGroupName(this.config.getSourceTopic()));
+ this.consumer.setConsumerGroup(this.taskId);
this.consumer.setNamesrvAddr(this.config.getSourceRocketmq());
+ List<TaskTopicInfo> topicList = JSONObject.parseArray(this.config.getTaskTopicList(), TaskTopicInfo.class);
+
try {
this.consumer.start();
- Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(this.config.getSourceTopic());
- if (!this.config.getQueueId().equals("")) {
- for (MessageQueue mq: mqs) {
- if (Integer.valueOf(this.config.getQueueId()) == mq.getQueueId()) {
+ for (TaskTopicInfo tti: topicList) {
+ Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(tti.getSourceTopic());
+ if (!tti.getQueueId().equals("")) {
+ // divide task by queue
+ for (MessageQueue mq: mqs) {
+ if (Integer.valueOf(tti.getQueueId()) == mq.getQueueId()) {
+ ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
+ ByteBuffer.wrap(RmqConstants.getPartition(
+ mq.getTopic(),
+ mq.getBrokerName(),
+ String.valueOf(mq.getQueueId())).getBytes("UTF-8")));
+
+ if (null != positionInfo && positionInfo.array().length > 0) {
+ String positionJson = new String(positionInfo.array(), "UTF-8");
+ JSONObject jsonObject = JSONObject.parseObject(positionJson);
+ this.config.setNextPosition(jsonObject.getLong(RmqConstants.NEXT_POSITION));
+ } else {
+ this.config.setNextPosition(0L);
+ }
+ mqOffsetMap.put(mq, this.config.getNextPosition());
+ }
+ }
+ } else {
+ // divide task by topic
+ for (MessageQueue mq: mqs) {
ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
- ByteBuffer.wrap(RmqConstants.getPartition(mq.getTopic(),
- mq.getBrokerName(), String.valueOf(mq.getQueueId()))
- .getBytes("UTF-8")));
+ ByteBuffer.wrap(RmqConstants.getPartition(
+ mq.getTopic(),
+ mq.getBrokerName(),
+ String.valueOf(mq.getQueueId())).getBytes("UTF-8")));
if (null != positionInfo && positionInfo.array().length > 0) {
String positionJson = new String(positionInfo.array(), "UTF-8");
JSONObject jsonObject = JSONObject.parseObject(positionJson);
this.config.setNextPosition(jsonObject.getLong(RmqConstants.NEXT_POSITION));
+ } else {
+ this.config.setNextPosition(0L);
}
mqOffsetMap.put(mq, this.config.getNextPosition());
}
}
- } else {
- for (MessageQueue mq: mqs) {
- ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
- ByteBuffer.wrap(RmqConstants.getPartition(mq.getTopic(),
- mq.getBrokerName(), String.valueOf(mq.getQueueId()))
- .getBytes("UTF-8")));
-
- if (null != positionInfo && positionInfo.array().length > 0) {
- String positionJson = new String(positionInfo.array(), "UTF-8");
- JSONObject jsonObject = JSONObject.parseObject(positionJson);
- this.config.setNextPosition(jsonObject.getLong(RmqConstants.NEXT_POSITION));
- }
- mqOffsetMap.put(mq, this.config.getNextPosition());
- }
}
-
+ started = true;
} catch (Exception e) {
- log.error("consumer of task {} start failed. ", this.taskId, e);
+ log.error("Consumer of task {} start failed.", this.taskId, e);
}
}
public void stop() {
- this.consumer.shutdown();
+
+ if (started) {
+ if (this.consumer != null) {
+ this.consumer.shutdown();
+ }
+ started = false;
+ }
}
public void pause() {
@@ -124,46 +144,48 @@ public class RmqSourceTask extends SourceTask {
private Collection<SourceDataEntry> pollCommonMessage() {
List<SourceDataEntry> res = new ArrayList<SourceDataEntry>();
-
- try {
- for (MessageQueue mq : this.mqOffsetMap.keySet()) {
- PullResult pullResult = consumer.pull(mq, "*", this.mqOffsetMap.get(mq), 32);
- switch (pullResult.getPullStatus()) {
- case FOUND: {
- this.mqOffsetMap.put(mq, pullResult.getNextBeginOffset());
- JSONObject jsonObject = new JSONObject();
- jsonObject.put(RmqConstants.NEXT_POSITION, pullResult.getNextBeginOffset());
-
- List<MessageExt> msgs = pullResult.getMsgFoundList();
- for (MessageExt m : msgs) {
+ if (started) {
+ try {
+ for (MessageQueue mq : this.mqOffsetMap.keySet()) {
+ PullResult pullResult = consumer.pull(mq, "*",
+ this.mqOffsetMap.get(mq), 32);
+ switch (pullResult.getPullStatus()) {
+ case FOUND: {
+ this.mqOffsetMap.put(mq, pullResult.getNextBeginOffset());
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put(RmqConstants.NEXT_POSITION, pullResult.getNextBeginOffset());
+ List<MessageExt> msgs = pullResult.getMsgFoundList();
Schema schema = new Schema();
schema.setDataSource(this.config.getSourceRocketmq());
- schema.setName(this.config.getSourceTopic());
+ schema.setName(mq.getTopic());
schema.setFields(new ArrayList<Field>());
- schema.getFields().add(new Field(0, FieldName.COMMON_MESSAGE.getKey(), FieldType.STRING));
+ schema.getFields().add(new Field(0,
+ FieldName.COMMON_MESSAGE.getKey(), FieldType.STRING));
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis())
.queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
- dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), JSONObject.toJSONString(m));
+ dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), JSONObject.toJSONString(msgs));
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap(RmqConstants.getPartition(this.config.getSourceTopic(),
- this.config.getBrokerName(),
- this.config.getQueueId()).getBytes("UTF-8")),
+ ByteBuffer.wrap(RmqConstants.getPartition(
+ mq.getTopic(),
+ mq.getBrokerName(),
+ String.valueOf(mq.getQueueId())).getBytes("UTF-8")),
ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8"))
);
res.add(sourceDataEntry);
+ break;
}
- break;
+ default:
+ break;
}
- default:
- break;
}
+ } catch (Exception e) {
+ log.error("Rocketmq replicator task poll error, current config: {}", JSON.toJSONString(config), e);
}
- } catch (Exception e) {
- log.error("Rocketmq connector task poll error, current config: {}", JSON.toJSONString(config), e);
+ } else {
+ log.warn("Rocketmq replicator task is not started.");
}
-
return res;
}
diff --git a/src/main/java/org/apache/rocketmq/connector/common/Utils.java b/src/main/java/org/apache/rocketmq/connector/common/ConstDefine.java
similarity index 70%
copy from src/main/java/org/apache/rocketmq/connector/common/Utils.java
copy to src/main/java/org/apache/rocketmq/connector/common/ConstDefine.java
index 71f538c..6e1eeac 100644
--- a/src/main/java/org/apache/rocketmq/connector/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/connector/common/ConstDefine.java
@@ -16,13 +16,9 @@
*/
package org.apache.rocketmq.connector.common;
-public class Utils {
+public class ConstDefine {
- public static String createGroupName(String prefix) {
- return new StringBuilder().append(prefix).append("_").append(System.currentTimeMillis()).toString();
- }
+ public static String TASK_GROUP_NAME_PREFIX = "REPLICATOR-TASK";
- public static String createTaskId(String prefix) {
- return new StringBuilder().append(prefix).append("_").append(System.currentTimeMillis()).toString();
- }
+ public static String REPLICATOR_ADMIN_PREFIX = "REPLICATOR-ADMIN";
}
diff --git a/src/main/java/org/apache/rocketmq/connector/common/Utils.java b/src/main/java/org/apache/rocketmq/connector/common/Utils.java
index 71f538c..f5822c6 100644
--- a/src/main/java/org/apache/rocketmq/connector/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/connector/common/Utils.java
@@ -19,10 +19,14 @@ package org.apache.rocketmq.connector.common;
public class Utils {
public static String createGroupName(String prefix) {
- return new StringBuilder().append(prefix).append("_").append(System.currentTimeMillis()).toString();
+ return new StringBuilder().append(prefix).append("@").append(System.currentTimeMillis()).toString();
+ }
+
+ public static String createGroupName(String prefix, String postfix) {
+ return new StringBuilder().append(prefix).append("@").append(postfix).toString();
}
public static String createTaskId(String prefix) {
- return new StringBuilder().append(prefix).append("_").append(System.currentTimeMillis()).toString();
+ return new StringBuilder().append(prefix).append("@").append(System.currentTimeMillis()).toString();
}
}
diff --git a/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java b/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java
index 64e9c94..13f8960 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java
+++ b/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java
@@ -21,28 +21,38 @@ import java.util.Set;
public class ConfigDefine {
- public static final String SOURCE_RMQ = "sourceRocketmq";
+ public static final String CONN_SOURCE_RMQ = "source-rocketmq";
- public static final String STORE_TOPIC = "storeTopic";
+ public static final String CONN_STORE_TOPIC = "replicator-store-topic";
- public static final String TARGET_RMQ = "targetRocketmq";
+ public static final String CONN_TARGET_RMQ = "target-rocketmq";
- public static final String DATA_TYPE = "dataType";
+ public static final String CONN_SOURCE_GROUP = "source-group";
- public static final String QUEUE_ID = "queueId";
+ public static final String CONN_DATA_TYPE = "data-type";
- public static final String TASK_DIVIDE_STRATEGY = "taskDivideStrategy";
+ public static final String CONN_TASK_DIVIDE_STRATEGY = "task-divide-strategy";
- public static final String BROKER_NAME = "brokerName";
+ public static final String CONN_BROKER_NAME = "broker-name";
- public static final String SOURCE_TOPIC = "sourceTopic";
+ public static final String CONN_SOURCE_TOPIC = "source-topic";
+ public static final String CONN_WHITE_LIST = "white-list";
+
+ public static final String CONN_SOURCE_RECORD_CONVERTER = "source-record-converter";
+
+ public static final String CONN_TASK_PARALLELISM = "task-parallelism";
+
+ /**
+ * The required key for all configurations.
+ */
public static final Set<String> REQUEST_CONFIG = new HashSet<String>(){
{
- add("sourceRocketmq");
- add("targetRocketmq");
- add("storeTopic");
- add("taskDivideStrategy");
+ add(CONN_SOURCE_RMQ);
+ add(CONN_TARGET_RMQ);
+ add(CONN_STORE_TOPIC);
+ add(CONN_WHITE_LIST);
+ add(CONN_SOURCE_RECORD_CONVERTER);
}
};
}
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java b/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
index 7bba3fb..d480b90 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
+++ b/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
@@ -19,27 +19,26 @@ package org.apache.rocketmq.connector.config;
public class TaskConfig {
private String storeTopic;
- private String sourceTopic;
+ private String sourceGroup;
private String sourceRocketmq;
private Integer dataType;
- private String brokerName;
- private String queueId;
private Long nextPosition;
+ private String taskTopicList;
- public String getStoreTopic() {
- return storeTopic;
+ public String getSourceGroup() {
+ return sourceGroup;
}
- public void setStoreTopic(String storeTopic) {
- this.storeTopic = storeTopic;
+ public void setSourceGroup(String sourceGroup) {
+ this.sourceGroup = sourceGroup;
}
- public String getSourceTopic() {
- return sourceTopic;
+ public String getStoreTopic() {
+ return storeTopic;
}
- public void setSourceTopic(String sourceTopic) {
- this.sourceTopic = sourceTopic;
+ public void setStoreTopic(String storeTopic) {
+ this.storeTopic = storeTopic;
}
public String getSourceRocketmq() {
@@ -62,22 +61,6 @@ public class TaskConfig {
this.dataType = dataType;
}
- public String getBrokerName() {
- return brokerName;
- }
-
- public void setBrokerName(String brokerName) {
- this.brokerName = brokerName;
- }
-
- public String getQueueId() {
- return queueId;
- }
-
- public void setQueueId(String queueId) {
- this.queueId = queueId;
- }
-
public Long getNextPosition() {
return nextPosition;
}
@@ -85,4 +68,12 @@ public class TaskConfig {
public void setNextPosition(Long nextPosition) {
this.nextPosition = nextPosition;
}
+
+ public String getTaskTopicList() {
+ return taskTopicList;
+ }
+
+ public void setTaskTopicList(String taskTopicList) {
+ this.taskTopicList = taskTopicList;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/connector/common/Utils.java b/src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java
similarity index 59%
copy from src/main/java/org/apache/rocketmq/connector/common/Utils.java
copy to src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java
index 71f538c..d8aa08a 100644
--- a/src/main/java/org/apache/rocketmq/connector/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java
@@ -14,15 +14,28 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.common;
+package org.apache.rocketmq.connector.config;
-public class Utils {
+public enum TaskConfigEnum {
- public static String createGroupName(String prefix) {
- return new StringBuilder().append(prefix).append("_").append(System.currentTimeMillis()).toString();
+ TASK_ID("taskId"),
+ TASK_SOURCE_GROUP("sourceGroup"),
+ TASK_SOURCE_ROCKETMQ("sourceRocketmq"),
+ TASK_SOURCE_TOPIC("sourceTopic"),
+ TASK_STORE_ROCKETMQ("storeTopic"),
+ TASK_DATA_TYPE("dataType"),
+ TASK_BROKER_NAME("brokerName"),
+ TASK_QUEUE_ID("queueId"),
+ TASK_NEXT_POSITION("nextPosition"),
+ TASK_TOPIC_INFO("taskTopicList");
+
+ private String key;
+
+ TaskConfigEnum(String key) {
+ this.key = key;
}
- public static String createTaskId(String prefix) {
- return new StringBuilder().append(prefix).append("_").append(System.currentTimeMillis()).toString();
+ public String getKey() {
+ return key;
}
}
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java b/src/main/java/org/apache/rocketmq/connector/config/TaskDivideConfig.java
similarity index 53%
copy from src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
copy to src/main/java/org/apache/rocketmq/connector/config/TaskDivideConfig.java
index 7bba3fb..7f904b3 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
+++ b/src/main/java/org/apache/rocketmq/connector/config/TaskDivideConfig.java
@@ -16,73 +16,64 @@
*/
package org.apache.rocketmq.connector.config;
-public class TaskConfig {
+public class TaskDivideConfig {
+
+ private String sourceNamesrvAddr;
private String storeTopic;
- private String sourceTopic;
- private String sourceRocketmq;
- private Integer dataType;
- private String brokerName;
- private String queueId;
- private Long nextPosition;
- public String getStoreTopic() {
- return storeTopic;
- }
+ private String srcRecordConverter;
- public void setStoreTopic(String storeTopic) {
- this.storeTopic = storeTopic;
- }
+ private int dataType;
- public String getSourceTopic() {
- return sourceTopic;
- }
+ private int taskParallelism;
- public void setSourceTopic(String sourceTopic) {
- this.sourceTopic = sourceTopic;
+ public TaskDivideConfig(String sourceNamesrvAddr, String storeTopic, String srcRecordConverter,
+ int dataType, int taskParallelism) {
+ this.sourceNamesrvAddr = sourceNamesrvAddr;
+ this.storeTopic = storeTopic;
+ this.srcRecordConverter = srcRecordConverter;
+ this.dataType = dataType;
+ this.taskParallelism = taskParallelism;
}
- public String getSourceRocketmq() {
- return sourceRocketmq;
+ public String getSourceNamesrvAddr() {
+ return sourceNamesrvAddr;
}
- public void setSourceRocketmq(String sourceRocketmq) {
- this.sourceRocketmq = sourceRocketmq;
+ public void setSourceNamesrvAddr(String sourceNamesrvAddr) {
+ this.sourceNamesrvAddr = sourceNamesrvAddr;
}
- public int getDataType() {
- return dataType;
- }
-
- public void setDataType(int dataType) {
- this.dataType = dataType;
+ public String getStoreTopic() {
+ return storeTopic;
}
- public void setDataType(Integer dataType) {
- this.dataType = dataType;
+ public void setStoreTopic(String storeTopic) {
+ this.storeTopic = storeTopic;
}
- public String getBrokerName() {
- return brokerName;
+ public String getSrcRecordConverter() {
+ return srcRecordConverter;
}
- public void setBrokerName(String brokerName) {
- this.brokerName = brokerName;
+ public void setSrcRecordConverter(String srcRecordConverter) {
+ this.srcRecordConverter = srcRecordConverter;
}
- public String getQueueId() {
- return queueId;
+ public int getDataType() {
+ return dataType;
}
- public void setQueueId(String queueId) {
- this.queueId = queueId;
+ public void setDataType(int dataType) {
+ this.dataType = dataType;
}
- public Long getNextPosition() {
- return nextPosition;
+ public int getTaskParallelism() {
+ return taskParallelism;
}
- public void setNextPosition(Long nextPosition) {
- this.nextPosition = nextPosition;
+ public void setTaskParallelism(int taskParallelism) {
+ this.taskParallelism = taskParallelism;
}
}
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java b/src/main/java/org/apache/rocketmq/connector/config/TaskTopicInfo.java
similarity index 61%
copy from src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
copy to src/main/java/org/apache/rocketmq/connector/config/TaskTopicInfo.java
index 7bba3fb..e1f47d5 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
+++ b/src/main/java/org/apache/rocketmq/connector/config/TaskTopicInfo.java
@@ -16,22 +16,16 @@
*/
package org.apache.rocketmq.connector.config;
-public class TaskConfig {
+public class TaskTopicInfo {
- private String storeTopic;
private String sourceTopic;
- private String sourceRocketmq;
- private Integer dataType;
private String brokerName;
private String queueId;
- private Long nextPosition;
- public String getStoreTopic() {
- return storeTopic;
- }
-
- public void setStoreTopic(String storeTopic) {
- this.storeTopic = storeTopic;
+ public TaskTopicInfo(String sourceTopic, String brokerName, String queueId) {
+ this.sourceTopic = sourceTopic;
+ this.brokerName = brokerName;
+ this.queueId = queueId;
}
public String getSourceTopic() {
@@ -42,26 +36,6 @@ public class TaskConfig {
this.sourceTopic = sourceTopic;
}
- public String getSourceRocketmq() {
- return sourceRocketmq;
- }
-
- public void setSourceRocketmq(String sourceRocketmq) {
- this.sourceRocketmq = sourceRocketmq;
- }
-
- public int getDataType() {
- return dataType;
- }
-
- public void setDataType(int dataType) {
- this.dataType = dataType;
- }
-
- public void setDataType(Integer dataType) {
- this.dataType = dataType;
- }
-
public String getBrokerName() {
return brokerName;
}
@@ -77,12 +51,4 @@ public class TaskConfig {
public void setQueueId(String queueId) {
this.queueId = queueId;
}
-
- public Long getNextPosition() {
- return nextPosition;
- }
-
- public void setNextPosition(Long nextPosition) {
- this.nextPosition = nextPosition;
- }
}
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java
index b6eae8f..caa69f3 100644
--- a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java
@@ -21,27 +21,19 @@ import io.openmessaging.internal.DefaultKeyValue;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.connector.config.ConfigDefine;
import org.apache.rocketmq.connector.config.DataType;
+import org.apache.rocketmq.connector.config.TaskDivideConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class DivideTaskByQueue extends TaskDivideStrategy {
- public List<KeyValue> divide(Map<String, List<MessageQueue>> topicRouteMap, String source, String storeTopic) {
+ public List<KeyValue> divide(Map<String, List<MessageQueue>> topicRouteMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
for (String t: topicRouteMap.keySet()) {
for (MessageQueue mq: topicRouteMap.get(t)) {
- KeyValue keyValue = new DefaultKeyValue();
- keyValue.put(ConfigDefine.STORE_TOPIC, storeTopic);
- keyValue.put(ConfigDefine.SOURCE_RMQ, source);
- keyValue.put(ConfigDefine.STORE_TOPIC, t);
- keyValue.put(ConfigDefine.BROKER_NAME, mq.getBrokerName());
- keyValue.put(ConfigDefine.QUEUE_ID, String.valueOf(mq.getQueueId()));
- keyValue.put(ConfigDefine.SOURCE_TOPIC, t);
- keyValue.put(ConfigDefine.DATA_TYPE, DataType.COMMON_MESSAGE.ordinal());
- config.add(keyValue);
}
}
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
index 7c96b28..53699a9 100644
--- a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
+++ b/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
@@ -16,31 +16,39 @@
*/
package org.apache.rocketmq.connector.strategy;
+import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.internal.DefaultKeyValue;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.connector.config.ConfigDefine;
-import org.apache.rocketmq.connector.config.DataType;
+import org.apache.rocketmq.connector.config.*;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class DivideTaskByTopic extends TaskDivideStrategy {
- public List<KeyValue> divide(Map<String, List<MessageQueue>> topicRouteMap, String source, String storeTopic) {
+ public List<KeyValue> divide(Map<String, List<MessageQueue>> topicRouteMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
-
+ int parallelism = tdc.getTaskParallelism();
+ int id = -1;
+ Map<Integer, List<TaskTopicInfo>> taskTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
for (String t: topicRouteMap.keySet()) {
+ int ind = ++id%parallelism;
+ if (!taskTopicList.containsKey(ind)) {
+ taskTopicList.put(ind, new ArrayList<TaskTopicInfo>());
+ }
+ taskTopicList.get(ind).add(new TaskTopicInfo(t, "", ""));
+ }
+
+ for (int i=0; i<parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
- keyValue.put(ConfigDefine.STORE_TOPIC, storeTopic);
- keyValue.put(ConfigDefine.SOURCE_RMQ, source);
- keyValue.put(ConfigDefine.STORE_TOPIC, t);
- keyValue.put(ConfigDefine.BROKER_NAME, "");
- keyValue.put(ConfigDefine.QUEUE_ID, "");
- keyValue.put(ConfigDefine.SOURCE_TOPIC, t);
- keyValue.put(ConfigDefine.DATA_TYPE, DataType.COMMON_MESSAGE.ordinal());
+ keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
+ keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
+ keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(taskTopicList.get(i)));
config.add(keyValue);
}
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java b/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java
index f847cb1..e80d092 100644
--- a/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java
+++ b/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java
@@ -18,11 +18,12 @@ package org.apache.rocketmq.connector.strategy;
import io.openmessaging.KeyValue;
import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.connector.config.TaskDivideConfig;
import java.util.List;
import java.util.Map;
public abstract class TaskDivideStrategy {
- public abstract List<KeyValue> divide(Map<String, List<MessageQueue>> topicMap, String source, String storeTopic);
+ public abstract List<KeyValue> divide(Map<String, List<MessageQueue>> topicMap, TaskDivideConfig tdc);
}
[rocketmq-connect] 10/39: Automatically create target topic. resolve #396
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit b4f8cd93fc6ee52a9034fa5b6f325f795d0b243f
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Tue Sep 10 23:07:49 2019 +0800
Automatically create target topic. resolve #396
---
.../rocketmq/replicator/RmqSourceReplicator.java | 97 ++++++++++++++++++----
.../apache/rocketmq/replicator/common/Utils.java | 59 ++++++++++++-
.../rocketmq/replicator/config/ConfigDefine.java | 6 +-
3 files changed, 142 insertions(+), 20 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index b49e06d..6ea3ae8 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -29,11 +29,14 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.body.TopicList;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.replicator.common.ConstDefine;
import org.apache.rocketmq.replicator.common.Utils;
import org.apache.rocketmq.replicator.config.ConfigDefine;
@@ -67,7 +70,8 @@ public class RmqSourceReplicator extends SourceConnector {
private int taskParallelism = 1;
- private DefaultMQAdminExt defaultMQAdminExt;
+ private DefaultMQAdminExt srcMQAdminExt;
+ private DefaultMQAdminExt targetMQAdminExt;
private volatile boolean adminStarted;
@@ -81,16 +85,26 @@ public class RmqSourceReplicator extends SourceConnector {
return;
}
RPCHook rpcHook = null;
- this.defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
- this.defaultMQAdminExt.setNamesrvAddr(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ));
- this.defaultMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- this.defaultMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ)));
+ this.srcMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ this.srcMQAdminExt.setNamesrvAddr(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ));
+ this.srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ)));
+
+ this.targetMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ this.targetMQAdminExt.setNamesrvAddr(this.replicatorConfig.getString(ConfigDefine.CONN_TARGET_RMQ));
+ this.targetMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ this.targetMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getString(ConfigDefine.CONN_TARGET_RMQ)));
+
try {
- defaultMQAdminExt.start();
- log.info("RocketMQ defaultMQAdminExt started");
+ this.srcMQAdminExt.start();
+ log.info("RocketMQ srcMQAdminExt started");
+
+ this.targetMQAdminExt.start();
+ log.info("RocketMQ targetMQAdminExt started");
} catch (MQClientException e) {
- log.error("Replicator start failed for `defaultMQAdminExt` exception.", e);
+ log.error("Replicator start failed for `srcMQAdminExt` exception.", e);
}
+
adminStarted = true;
}
@@ -172,14 +186,16 @@ public class RmqSourceReplicator extends SourceConnector {
}
public void buildRoute() {
+ List<Pattern> patterns = new ArrayList<Pattern>();
+ String srcCluster = this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_CLUSTER);
try {
- List<Pattern> patterns = new ArrayList<Pattern>();
+ Set<String> targetTopicSet = fetchTargetTopics();
for (String topic : this.whiteList) {
Pattern pattern = Pattern.compile(topic);
patterns.add(pattern);
}
- TopicList topics = defaultMQAdminExt.fetchAllTopicList();
+ TopicList topics = srcMQAdminExt.fetchAllTopicList();
for (String topic : topics.getTopicList()) {
if ((syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) ||
(syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) ||
@@ -188,14 +204,29 @@ public class RmqSourceReplicator extends SourceConnector {
for (Pattern pattern : patterns) {
Matcher matcher = pattern.matcher(topic);
if (matcher.matches()) {
- TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+ if (!targetTopicSet.contains(topic)) {
+ ensureTargetTopic(topic, topic);
+ }
+
+ // different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
+ // QueueData use brokerName as unique info on cluster of rocketmq. so when we want to get QueueData of
+ // expected cluster, we should get brokerNames of expected cluster, and then filter queueDatas.
+ List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, topic, srcCluster);
+ Set<String> brokerNameSet = new HashSet<String>();
+ for (BrokerData b : brokerList) {
+ brokerNameSet.add(b.getBrokerName());
+ }
+
+ TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
if (!topicRouteMap.containsKey(topic)) {
topicRouteMap.put(topic, new ArrayList<MessageQueue>());
}
for (QueueData qd : topicRouteData.getQueueDatas()) {
- for (int i = 0; i < qd.getReadQueueNums(); i++) {
- MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
- topicRouteMap.get(topic).add(mq);
+ if (brokerNameSet.contains(qd.getBrokerName())) {
+ for (int i = 0; i < qd.getReadQueueNums(); i++) {
+ MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
+ topicRouteMap.get(topic).add(mq);
+ }
}
}
}
@@ -205,7 +236,7 @@ public class RmqSourceReplicator extends SourceConnector {
} catch (Exception e) {
log.error("Fetch topic list error.", e);
} finally {
- defaultMQAdminExt.shutdown();
+ srcMQAdminExt.shutdown();
}
}
@@ -216,5 +247,41 @@ public class RmqSourceReplicator extends SourceConnector {
public Map<String, List<MessageQueue>> getTopicRouteMap() {
return this.topicRouteMap;
}
+
+ public Set<String> fetchTargetTopics() throws RemotingException, MQClientException, InterruptedException {
+ String targetCluster = this.replicatorConfig.getString(ConfigDefine.CONN_TARGET_CLUSTER);
+ TopicList targetTopics = this.targetMQAdminExt.fetchTopicsByCLuster(targetCluster);
+ return targetTopics.getTopicList();
+ }
+
+ /**
+ * ensure target topic eixst. if target topic does not exist, ensureTopic will create target topic on target
+ * cluster, with same TopicConfig but using target topic name. any exception will be caught and then throw
+ * IllegalStateException.
+ *
+ * @param srcTopic
+ * @param targetTopic
+ * @throws RemotingException
+ * @throws MQClientException
+ * @throws InterruptedException
+ */
+ public void ensureTargetTopic(String srcTopic,
+ String targetTopic) throws RemotingException, MQClientException, InterruptedException {
+ String srcCluster = this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_CLUSTER);
+ String targetCluster = this.replicatorConfig.getString(ConfigDefine.CONN_TARGET_CLUSTER);
+
+ List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, srcTopic, srcCluster);
+ if (brokerList.size() == 0) {
+ throw new IllegalStateException(String.format("no broker found for srcTopic: %s srcCluster: %s", srcTopic, srcCluster));
+ }
+
+ String brokerAddr = brokerList.get(0).selectBrokerAddr();
+ TopicConfig topicConfig = this.srcMQAdminExt.examineTopicConfig(brokerAddr, srcTopic);
+ topicConfig.setTopicName(targetTopic);
+ Utils.createTopic(this.targetMQAdminExt, topicConfig, targetCluster);
+
+ throw new IllegalStateException("");
+ }
+
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index 6888038..e5c0866 100644
--- a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
@@ -19,8 +19,20 @@ package org.apache.rocketmq.replicator.common;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class Utils {
+ private static final Logger log = LoggerFactory.getLogger(Utils.class);
public static String createGroupName(String prefix) {
return new StringBuilder().append(prefix).append("-").append(System.currentTimeMillis()).toString();
@@ -37,7 +49,7 @@ public class Utils {
public static String createInstanceName(String namesrvAddr) {
String[] namesrvArray = namesrvAddr.split(";");
List<String> namesrvList = new ArrayList<String>();
- for (String ns: namesrvArray) {
+ for (String ns : namesrvArray) {
if (!namesrvList.contains(ns)) {
namesrvList.add(ns);
}
@@ -45,4 +57,49 @@ public class Utils {
Collections.sort(namesrvList);
return String.valueOf(namesrvList.toString().hashCode());
}
+
+
+ public static List<BrokerData> examineBrokerData(DefaultMQAdminExt defaultMQAdminExt, String topic, String cluster) throws RemotingException, MQClientException, InterruptedException {
+ List<BrokerData> brokerList = new ArrayList<BrokerData>();
+
+ TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+ if (topicRouteData.getBrokerDatas() != null) { // check下
+ for (BrokerData broker : topicRouteData.getBrokerDatas()) {
+ if (StringUtils.equals(broker.getCluster(), cluster)) {
+ brokerList.add(broker);
+ }
+ }
+ }
+ return brokerList;
+ }
+
+ public static void createTopic(DefaultMQAdminExt defaultMQAdminExt, TopicConfig topicConfig, String clusterName) {
+ try {
+ Set<String> masterSet =
+ CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
+ for (String addr : masterSet) {
+ defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig);
+ log.info("create topic to %s success.%n", addr);
+ }
+
+ if (topicConfig.isOrder()) {
+ Set<String> brokerNameSet =
+ CommandUtil.fetchBrokerNameByClusterName(defaultMQAdminExt, clusterName);
+ StringBuilder orderConf = new StringBuilder();
+ String splitor = "";
+ for (String s : brokerNameSet) {
+ orderConf.append(splitor).append(s).append(":")
+ .append(topicConfig.getWriteQueueNums());
+ splitor = ";";
+ }
+ defaultMQAdminExt.createOrUpdateOrderConf(topicConfig.getTopicName(),
+ orderConf.toString(), true);
+ log.info("set cluster orderConf=[%s]", orderConf);
+ }
+
+ return;
+ } catch (Exception e) {
+ throw new IllegalArgumentException("create topic: " + topicConfig + "failed", e);
+ }
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
index 3934c2f..ddf4972 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
@@ -22,10 +22,12 @@ import java.util.Set;
public class ConfigDefine {
public static final String CONN_SOURCE_RMQ = "source-rocketmq";
+ public static final String CONN_SOURCE_CLUSTER = "source-cluster";
public static final String CONN_STORE_TOPIC = "replicator-store-topic";
public static final String CONN_TARGET_RMQ = "target-rocketmq";
+ public static final String CONN_TARGET_CLUSTER = "target-cluster";
public static final String CONN_SOURCE_GROUP = "source-group";
@@ -33,10 +35,6 @@ public class ConfigDefine {
public static final String CONN_TASK_DIVIDE_STRATEGY = "task-divide-strategy";
- public static final String CONN_BROKER_NAME = "broker-name";
-
- public static final String CONN_SOURCE_TOPIC = "source-topic";
-
public static final String CONN_WHITE_LIST = "white-list";
public static final String CONN_SOURCE_RECORD_CONVERTER = "source-record-converter";
[rocketmq-connect] 28/39: [ISSUE #492] Optimize metadata synchronization and fix RocketMQConverter bug (#493)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 070dd98cf2492fd760dc2d34b1f18057e1b2a56f
Author: zhoubo <87...@qq.com>
AuthorDate: Thu Dec 26 11:07:45 2019 +0800
[ISSUE #492] Optimize metadata synchronization and fix RocketMQConverter bug (#493)
* TopicList is null exception and frequent requestTaskReconfiguration
* https://github.com/apache/rocketmq-externals/issues/478
* * Runtime add some log
* Fix replicator add new topic frequent requestTaskReconfiguration bug
* Optimize metadata synchronization and fix RocketMQConverter bug
https://github.com/apache/rocketmq-externals/issues/492
* Fix replicator bug
* Exclude groups starting with RebalanceService
---
.../org/apache/rocketmq/replicator/MetaSourceTask.java | 1 +
.../apache/rocketmq/replicator/RmqMetaReplicator.java | 4 +++-
.../apache/rocketmq/replicator/RmqSourceReplicator.java | 16 +++++++++-------
3 files changed, 13 insertions(+), 8 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
index 16bf464..f28078f 100644
--- a/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
@@ -77,6 +77,7 @@ public class MetaSourceTask extends SourceTask {
if (started) {
started = false;
}
+ srcMQAdminExt.shutdown();
}
@Override public void pause() {
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
index 3b00de5..75bc919 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
@@ -106,6 +106,8 @@ public class RmqMetaReplicator extends SourceConnector {
@Override public void stop() {
log.info("stopping...");
this.executor.shutdown();
+ this.srcMQAdminExt.shutdown();
+ this.targetMQAdminExt.shutdown();
}
@Override public void pause() {
@@ -217,7 +219,7 @@ public class RmqMetaReplicator extends SourceConnector {
private boolean skipInnerGroup(String group) {
if (INNER_CONSUMER_GROUPS.contains(group) || group.startsWith("CID_RMQ_SYS_") || group.startsWith("PositionManage") ||
- group.startsWith("ConfigManage") || group.startsWith("OffsetManage")) {
+ group.startsWith("ConfigManage") || group.startsWith("OffsetManage") || group.startsWith("DefaultConnectCluster") || group.startsWith("RebalanceService")) {
return false;
}
return true;
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index 82744ed..10efae6 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -242,7 +242,7 @@ public class RmqSourceReplicator extends SourceConnector {
Matcher matcher = pattern.matcher(topic);
if (matcher.matches()) {
String targetTopic = generateTargetTopic(topic);
- if (!targetTopicSet.contains(topic)) {
+ if (!targetTopicSet.contains(targetTopic)) {
ensureTargetTopic(topic, targetTopic);
}
@@ -273,8 +273,6 @@ public class RmqSourceReplicator extends SourceConnector {
}
} catch (Exception e) {
log.error("Fetch topic list error.", e);
- } finally {
- srcMQAdminExt.shutdown();
}
}
@@ -309,12 +307,16 @@ public class RmqSourceReplicator extends SourceConnector {
throw new IllegalStateException(String.format("no broker found for srcTopic: %s srcCluster: %s", srcTopic, srcCluster));
}
- String brokerAddr = brokerList.get(0).selectBrokerAddr();
- TopicConfig topicConfig = this.srcMQAdminExt.examineTopicConfig(brokerAddr, srcTopic);
+ final TopicRouteData topicRouteData = this.srcMQAdminExt.examineTopicRouteInfo(srcTopic);
+ final TopicConfig topicConfig = new TopicConfig();
+ final List<QueueData> queueDatas = topicRouteData.getQueueDatas();
+ QueueData queueData = queueDatas.get(0);
+ topicConfig.setPerm(queueData.getPerm());
+ topicConfig.setReadQueueNums(queueData.getReadQueueNums());
+ topicConfig.setWriteQueueNums(queueData.getWriteQueueNums());
+ topicConfig.setTopicSysFlag(queueData.getTopicSynFlag());
topicConfig.setTopicName(targetTopic);
Utils.createTopic(this.targetMQAdminExt, topicConfig, targetCluster);
-
- throw new IllegalStateException("");
}
public String generateTargetTopic(String topic) {
[rocketmq-connect] 13/39: add DivideTaskByQueue. resolve #397
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 90541a271a54f6e392ce146cb39666c2466a5375
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Fri Sep 20 22:25:26 2019 +0800
add DivideTaskByQueue. resolve #397
---
.../rocketmq/replicator/config/TaskTopicInfo.java | 2 +-
.../replicator/strategy/DivideTaskByQueue.java | 25 ++++++++++++++++++++--
2 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
index 1086295..3e2962b 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
@@ -18,7 +18,7 @@ package org.apache.rocketmq.replicator.config;
import org.apache.rocketmq.common.message.MessageQueue;
-public class TaskTopicInfo extends MessageQueue{
+public class TaskTopicInfo extends MessageQueue {
private String targetTopic;
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
index d6a15ad..d909873 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
@@ -16,8 +16,13 @@
*/
package org.apache.rocketmq.replicator.strategy;
+import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
+import io.openmessaging.internal.DefaultKeyValue;
+import java.util.HashMap;
import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.replicator.config.DataType;
+import org.apache.rocketmq.replicator.config.TaskConfigEnum;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import java.util.ArrayList;
import java.util.List;
@@ -28,13 +33,29 @@ public class DivideTaskByQueue extends TaskDivideStrategy {
public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
-
+ int parallelism = tdc.getTaskParallelism();
+ Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
+ int id = -1;
for (String t : topicRouteMap.keySet()) {
for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
+ int ind = ++id % parallelism;
+ if (!queueTopicList.containsKey(ind)) {
+ queueTopicList.put(ind, new ArrayList<TaskTopicInfo>());
+ }
+ queueTopicList.get(ind).add(taskTopicInfo);
}
}
- return config;
+ for (int i = 0; i < parallelism; i++) {
+ KeyValue keyValue = new DefaultKeyValue();
+ keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
+ keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
+ keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
+ config.add(keyValue);
+ }
+ return config;
}
}
[rocketmq-connect] 35/39: [rocketmq-replicator] Support ACL (#832)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 1ca3a0731fc581761693e9c83aac691e0f52d55d
Author: Git_Yang <30...@users.noreply.github.com>
AuthorDate: Wed Oct 20 14:27:16 2021 +0800
[rocketmq-replicator] Support ACL (#832)
Signed-off-by: zhangyang21 <zh...@xiaomi.com>
---
.../apache/rocketmq/replicator/MetaSourceTask.java | 24 ++++------
.../rocketmq/replicator/RmqMetaReplicator.java | 34 +++++++------
.../rocketmq/replicator/RmqSourceReplicator.java | 46 ++++++++----------
.../apache/rocketmq/replicator/RmqSourceTask.java | 19 ++++++--
.../rocketmq/replicator/common/ConstDefine.java | 5 ++
.../apache/rocketmq/replicator/common/Utils.java | 55 ++++++++++++++++++----
.../rocketmq/replicator/config/ConfigDefine.java | 6 +++
.../replicator/config/RmqConnectorConfig.java | 42 +++++++++++++++++
.../rocketmq/replicator/config/TaskConfig.java | 29 +++++++++++-
.../rocketmq/replicator/config/TaskConfigEnum.java | 5 +-
.../replicator/config/TaskDivideConfig.java | 43 ++++++++++++++++-
.../strategy/DivideTaskByConsistentHash.java | 3 ++
.../replicator/strategy/DivideTaskByQueue.java | 3 ++
.../replicator/strategy/DivideTaskByTopic.java | 3 ++
14 files changed, 244 insertions(+), 73 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
index f28078f..a66ba22 100644
--- a/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
@@ -38,7 +38,6 @@ import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.admin.ConsumeStats;
import org.apache.rocketmq.common.admin.OffsetWrapper;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.replicator.common.ConstDefine;
import org.apache.rocketmq.replicator.common.Utils;
import org.apache.rocketmq.replicator.config.ConfigUtil;
import org.apache.rocketmq.replicator.config.TaskConfig;
@@ -64,10 +63,16 @@ public class MetaSourceTask extends SourceTask {
this.taskId = Utils.createTaskId(Thread.currentThread().getName());
}
- @Override public void start(KeyValue config) {
+ @Override
+ public void start(KeyValue config) {
ConfigUtil.load(config, this.config);
- startAdmin();
+ try {
+ this.srcMQAdminExt = Utils.startMQAdminTool(this.config);
+ } catch (MQClientException e) {
+ log.error("Replicator task start failed for `startMQAdminTool` exception.", e);
+ throw new IllegalStateException("Replicator task start failed for `startMQAdminTool` exception.");
+ }
this.store = new OffsetSyncStore(this.srcMQAdminExt, this.config);
this.started = true;
@@ -148,17 +153,4 @@ public class MetaSourceTask extends SourceTask {
}
return res;
}
-
- private void startAdmin() {
- this.srcMQAdminExt = new DefaultMQAdminExt();
- this.srcMQAdminExt.setNamesrvAddr(this.config.getSourceRocketmq());
- this.srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(this.config.getSourceRocketmq()));
- try {
- this.srcMQAdminExt.start();
- } catch (MQClientException e) {
- log.error("start src mq admin failed.", e);
- throw new IllegalStateException("start src mq admin failed");
- }
- }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
index 856fce8..e645b6c 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
@@ -96,9 +96,16 @@ public class RmqMetaReplicator extends SourceConnector {
return "";
}
- @Override public void start() {
+ @Override
+ public void start() {
log.info("starting...");
- startMQAdminTools();
+ try {
+ startMQAdminTools();
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `startMQAdminTools` exception.", e);
+ return;
+ }
+
executor.scheduleAtFixedRate(this::refreshConsumerGroups, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
executor.scheduleAtFixedRate(this::syncSubConfig, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
}
@@ -122,13 +129,19 @@ public class RmqMetaReplicator extends SourceConnector {
return MetaSourceTask.class;
}
- @Override public List<KeyValue> taskConfigs() {
+ @Override
+ public List<KeyValue> taskConfigs() {
log.debug("preparing taskConfig...");
if (!configValid) {
return new ArrayList<>();
}
- startMQAdminTools();
+ try {
+ startMQAdminTools();
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `startMQAdminTools` exception.", e);
+ throw new IllegalStateException("Replicator start failed for `startMQAdminTools` exception.");
+ }
try {
this.syncSubConfig();
@@ -140,20 +153,13 @@ public class RmqMetaReplicator extends SourceConnector {
return Utils.groupPartitions(new ArrayList<>(this.knownGroups), this.replicatorConfig.getTaskParallelism(), replicatorConfig);
}
- private synchronized void startMQAdminTools() {
+ private synchronized void startMQAdminTools() throws MQClientException {
if (!configValid || adminStarted) {
return;
}
- try {
- ImmutablePair<DefaultMQAdminExt, DefaultMQAdminExt> pair = Utils.startMQAdminTools(this.replicatorConfig);
- this.srcMQAdminExt = pair.getLeft();
- this.targetMQAdminExt = pair.getRight();
- log.info("RocketMQ targetMQAdminExt started");
- } catch (MQClientException e) {
- log.error("Replicator start failed for `srcMQAdminExt` exception.", e);
- }
-
+ this.srcMQAdminExt = Utils.startSrcMQAdminTool(this.replicatorConfig);
+ this.targetMQAdminExt = Utils.startTargetMQAdminTool(this.replicatorConfig);
adminStarted = true;
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index 10efae6..50f4a17 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -40,9 +40,7 @@ import org.apache.rocketmq.common.protocol.body.TopicList;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
-import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.exception.RemotingException;
-import org.apache.rocketmq.replicator.common.ConstDefine;
import org.apache.rocketmq.replicator.common.Utils;
import org.apache.rocketmq.replicator.config.ConfigDefine;
import org.apache.rocketmq.replicator.config.DataType;
@@ -80,31 +78,13 @@ public class RmqSourceReplicator extends SourceConnector {
executor = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("RmqSourceReplicator-SourceWatcher-%d").daemon(true).build());
}
- private synchronized void startMQAdminTools() {
+ private synchronized void startMQAdminTools() throws MQClientException {
if (!configValid || adminStarted) {
return;
}
- RPCHook rpcHook = null;
- this.srcMQAdminExt = new DefaultMQAdminExt(rpcHook);
- this.srcMQAdminExt.setNamesrvAddr(this.replicatorConfig.getSrcNamesrvs());
- this.srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getSrcNamesrvs()));
-
- this.targetMQAdminExt = new DefaultMQAdminExt(rpcHook);
- this.targetMQAdminExt.setNamesrvAddr(this.replicatorConfig.getTargetNamesrvs());
- this.targetMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- this.targetMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getTargetNamesrvs()));
-
- try {
- this.srcMQAdminExt.start();
- log.info("RocketMQ srcMQAdminExt started");
-
- this.targetMQAdminExt.start();
- log.info("RocketMQ targetMQAdminExt started");
- } catch (MQClientException e) {
- log.error("Replicator start failed for `srcMQAdminExt` exception.", e);
- }
+ this.srcMQAdminExt = Utils.startSrcMQAdminTool(this.replicatorConfig);
+ this.targetMQAdminExt = Utils.startTargetMQAdminTool(this.replicatorConfig);
adminStarted = true;
}
@@ -129,7 +109,13 @@ public class RmqSourceReplicator extends SourceConnector {
@Override
public void start() {
- startMQAdminTools();
+ try {
+ startMQAdminTools();
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `startMQAdminTools` exception.", e);
+ return;
+ }
+
buildRoute();
startListner();
}
@@ -207,7 +193,12 @@ public class RmqSourceReplicator extends SourceConnector {
return new ArrayList<KeyValue>();
}
- startMQAdminTools();
+ try {
+ startMQAdminTools();
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `startMQAdminTools` exception.", e);
+ throw new IllegalStateException("Replicator start failed for `startMQAdminTools` exception.");
+ }
buildRoute();
@@ -217,7 +208,10 @@ public class RmqSourceReplicator extends SourceConnector {
this.replicatorConfig.getStoreTopic(),
this.replicatorConfig.getConverter(),
DataType.COMMON_MESSAGE.ordinal(),
- this.replicatorConfig.getTaskParallelism()
+ this.replicatorConfig.getTaskParallelism(),
+ this.replicatorConfig.isSrcAclEnable(),
+ this.replicatorConfig.getSrcAccessKey(),
+ this.replicatorConfig.getSrcSecretKey()
);
return this.replicatorConfig.getTaskDivideStrategy().divide(this.topicRouteMap, tdc);
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index 87ed9a8..ca7edb4 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -35,10 +35,13 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.replicator.common.Utils;
import org.apache.rocketmq.replicator.config.ConfigUtil;
import org.apache.rocketmq.replicator.config.DataType;
@@ -57,7 +60,7 @@ public class RmqSourceTask extends SourceTask {
private final String taskId;
private final TaskConfig config;
- private final DefaultMQPullConsumer consumer;
+ private DefaultMQPullConsumer consumer;
private volatile boolean started = false;
private final long TIMEOUT = 1000 * 60 * 10;
private final long WAIT_TIME = 1000 * 2;
@@ -66,11 +69,11 @@ public class RmqSourceTask extends SourceTask {
public RmqSourceTask() {
this.config = new TaskConfig();
- this.consumer = new DefaultMQPullConsumer();
this.taskId = Utils.createTaskId(Thread.currentThread().getName());
mqOffsetMap = new HashMap<>();
}
+ @Override
public Collection<SourceDataEntry> poll() {
if (this.config.getDataType() == DataType.COMMON_MESSAGE.ordinal()) {
@@ -84,8 +87,14 @@ public class RmqSourceTask extends SourceTask {
}
}
+ @Override
public void start(KeyValue config) {
ConfigUtil.load(config, this.config);
+ RPCHook rpcHook = null;
+ if (this.config.isSrcAclEnable()) {
+ rpcHook = new AclClientRPCHook(new SessionCredentials(this.config.getSrcAccessKey(), this.config.getSrcSecretKey()));
+ }
+ this.consumer = new DefaultMQPullConsumer(rpcHook);
this.consumer.setConsumerGroup(this.taskId);
this.consumer.setNamesrvAddr(this.config.getSourceRocketmq());
this.consumer.setInstanceName(Utils.createInstanceName(this.config.getSourceRocketmq()));
@@ -113,11 +122,13 @@ public class RmqSourceTask extends SourceTask {
started = true;
} catch (Exception e) {
log.error("Consumer of task {} start failed.", this.taskId, e);
+ throw new IllegalStateException(String.format("Consumer of task %s start failed.", this.taskId));
}
log.info("RocketMQ source task started");
}
- @Override public void stop() {
+ @Override
+ public void stop() {
if (started) {
if (this.consumer != null) {
@@ -127,10 +138,12 @@ public class RmqSourceTask extends SourceTask {
}
}
+ @Override
public void pause() {
}
+ @Override
public void resume() {
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java b/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
index 08d985b..83583ca 100644
--- a/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
@@ -21,4 +21,9 @@ public class ConstDefine {
public static String TASK_GROUP_NAME_PREFIX = "REPLICATOR-TASK";
public static String REPLICATOR_ADMIN_PREFIX = "REPLICATOR-ADMIN";
+
+ public static String REPLICATOR_ADMIN_GROUP = "REPLICATOR-ADMIN-GROUP";
+
+ public static String REPLICATOR_TASK_ADMIN_GROUP = "REPLICATOR-TASK-ADMIN-GROUP";
+
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index 9ccee41..30dc214 100644
--- a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
@@ -23,9 +23,11 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.rocketmq.acl.common.AclClientRPCHook;
+import org.apache.rocketmq.acl.common.SessionCredentials;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.protocol.route.BrokerData;
@@ -34,6 +36,7 @@ import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.replicator.config.DataType;
import org.apache.rocketmq.replicator.config.RmqConnectorConfig;
+import org.apache.rocketmq.replicator.config.TaskConfig;
import org.apache.rocketmq.replicator.config.TaskConfigEnum;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.apache.rocketmq.tools.command.CommandUtil;
@@ -67,6 +70,10 @@ public class Utils {
return String.valueOf(namesrvList.toString().hashCode());
}
+ public static String createUniqIntanceName(String prefix) {
+ return new StringBuilder(prefix).append("-").append(UUID.randomUUID().toString()).toString();
+ }
+
public static List<BrokerData> examineBrokerData(DefaultMQAdminExt defaultMQAdminExt, String topic,
String cluster) throws RemotingException, MQClientException, InterruptedException {
List<BrokerData> brokerList = new ArrayList<>();
@@ -144,25 +151,53 @@ public class Utils {
return result;
}
- public static ImmutablePair<DefaultMQAdminExt, DefaultMQAdminExt> startMQAdminTools(
+ public static DefaultMQAdminExt startSrcMQAdminTool(
RmqConnectorConfig replicatorConfig) throws MQClientException {
RPCHook rpcHook = null;
+ if (replicatorConfig.isSrcAclEnable()) {
+ rpcHook = new AclClientRPCHook(new SessionCredentials(replicatorConfig.getSrcAccessKey(), replicatorConfig.getSrcSecretKey()));
+ }
DefaultMQAdminExt srcMQAdminExt = new DefaultMQAdminExt(rpcHook);
srcMQAdminExt.setNamesrvAddr(replicatorConfig.getSrcNamesrvs());
- srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- srcMQAdminExt.setInstanceName(Utils.createInstanceName(replicatorConfig.getSrcNamesrvs()));
+ srcMQAdminExt.setAdminExtGroup(ConstDefine.REPLICATOR_ADMIN_GROUP);
+ srcMQAdminExt.setInstanceName(Utils.createUniqIntanceName(replicatorConfig.getSrcNamesrvs()));
+
+ srcMQAdminExt.start();
+ log.info("SOURCE: RocketMQ srcMQAdminExt started");
+ return srcMQAdminExt;
+ }
+
+ public static DefaultMQAdminExt startTargetMQAdminTool(
+ RmqConnectorConfig replicatorConfig) throws MQClientException {
+ RPCHook rpcHook = null;
+ if (replicatorConfig.isTargetAclEnable()) {
+ rpcHook = new AclClientRPCHook(new SessionCredentials(replicatorConfig.getTargetAccessKey(), replicatorConfig.getTargetSecretKey()));
+ }
DefaultMQAdminExt targetMQAdminExt = new DefaultMQAdminExt(rpcHook);
targetMQAdminExt.setNamesrvAddr(replicatorConfig.getTargetNamesrvs());
- targetMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- targetMQAdminExt.setInstanceName(Utils.createInstanceName(replicatorConfig.getTargetNamesrvs()));
+ targetMQAdminExt.setAdminExtGroup(ConstDefine.REPLICATOR_ADMIN_GROUP);
+ targetMQAdminExt.setInstanceName(Utils.createUniqIntanceName(replicatorConfig.getTargetNamesrvs()));
- srcMQAdminExt.start();
- log.info("RocketMQ srcMQAdminExt started");
+ targetMQAdminExt.start();
+ log.info("TARGET: RocketMQ targetMQAdminExt started.");
+
+ return targetMQAdminExt;
+ }
+
+ public static DefaultMQAdminExt startMQAdminTool(TaskConfig taskConfig) throws MQClientException {
+ RPCHook rpcHook = null;
+ if (taskConfig.isSrcAclEnable()) {
+ rpcHook = new AclClientRPCHook(new SessionCredentials(taskConfig.getSrcAccessKey(), taskConfig.getSrcSecretKey()));
+ }
+ DefaultMQAdminExt targetMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ targetMQAdminExt.setNamesrvAddr(taskConfig.getSourceRocketmq());
+ targetMQAdminExt.setAdminExtGroup(ConstDefine.REPLICATOR_TASK_ADMIN_GROUP);
+ targetMQAdminExt.setInstanceName(Utils.createUniqIntanceName(taskConfig.getSourceRocketmq()));
targetMQAdminExt.start();
- log.info("RocketMQ targetMQAdminExt started");
+ log.info("TARGET: RocketMQ targetMQAdminExt started.");
- return ImmutablePair.of(srcMQAdminExt, targetMQAdminExt);
+ return targetMQAdminExt;
}
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
index a3514ee..b4de086 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
@@ -23,11 +23,17 @@ public class ConfigDefine {
public static final String CONN_SOURCE_RMQ = "source-rocketmq";
public static final String CONN_SOURCE_CLUSTER = "source-cluster";
+ public static final String CONN_SOURCE_ACL_ENBALE = "source-acl-enable";
+ public static final String CONN_SOURCE_ACCESS_KEY = "source-access-key";
+ public static final String CONN_SOURCE_SECRET_KEY = "source-secret-key";
public static final String CONN_STORE_TOPIC = "replicator-store-topic";
public static final String CONN_TARGET_RMQ = "target-rocketmq";
public static final String CONN_TARGET_CLUSTER = "target-cluster";
+ public static final String CONN_TARGET_ACL_ENBALE = "target-acl-enable";
+ public static final String CONN_TARGET_ACCESS_KEY = "target-access-key";
+ public static final String CONN_TARGET_SECRET_KEY = "target-secret-key";
public static final String CONN_SOURCE_GROUP = "source-group";
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java b/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
index 395d75d..2d4fc2a 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
@@ -38,6 +38,12 @@ public class RmqConnectorConfig {
private long refreshInterval;
private String renamePattern;
private String offsetSyncTopic;
+ private boolean srcAclEnable = false;
+ private String srcAccessKey;
+ private String srcSecretKey;
+ private boolean targetAclEnable = false;
+ private String targetAccessKey;
+ private String targetSecretKey;
public RmqConnectorConfig() {
}
@@ -64,6 +70,18 @@ public class RmqConnectorConfig {
refreshInterval = config.getLong(ConfigDefine.REFRESH_INTERVAL, 3);
renamePattern = config.getString(ConfigDefine.CONN_TOPIC_RENAME_FMT);
offsetSyncTopic = config.getString(ConfigDefine.OFFSET_SYNC_TOPIC);
+
+ if (config.containsKey(ConfigDefine.CONN_SOURCE_ACL_ENBALE)) {
+ srcAclEnable = Boolean.parseBoolean(config.getString(ConfigDefine.CONN_SOURCE_ACL_ENBALE));
+ srcAccessKey = config.getString(ConfigDefine.CONN_SOURCE_ACCESS_KEY);
+ srcSecretKey = config.getString(ConfigDefine.CONN_SOURCE_SECRET_KEY);
+ }
+
+ if (config.containsKey(ConfigDefine.CONN_TARGET_ACL_ENBALE)) {
+ targetAclEnable = Boolean.parseBoolean(config.getString(ConfigDefine.CONN_TARGET_ACL_ENBALE));
+ targetAccessKey = config.getString(ConfigDefine.CONN_TARGET_ACCESS_KEY);
+ targetSecretKey = config.getString(ConfigDefine.CONN_TARGET_SECRET_KEY);
+ }
}
private void buildWhiteList(KeyValue config) {
@@ -127,4 +145,28 @@ public class RmqConnectorConfig {
public String getOffsetSyncTopic() {
return this.offsetSyncTopic;
}
+
+ public boolean isSrcAclEnable() {
+ return srcAclEnable;
+ }
+
+ public String getSrcAccessKey() {
+ return srcAccessKey;
+ }
+
+ public String getSrcSecretKey() {
+ return srcSecretKey;
+ }
+
+ public boolean isTargetAclEnable() {
+ return targetAclEnable;
+ }
+
+ public String getTargetAccessKey() {
+ return targetAccessKey;
+ }
+
+ public String getTargetSecretKey() {
+ return targetSecretKey;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
index 15676ce..7921585 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
@@ -16,8 +16,6 @@
*/
package org.apache.rocketmq.replicator.config;
-import java.util.List;
-
public class TaskConfig {
private String sourceCluster;
@@ -29,6 +27,9 @@ public class TaskConfig {
private String taskTopicList;
private String taskGroupList;
private String offsetSyncTopic;
+ private boolean srcAclEnable = false;
+ private String srcAccessKey;
+ private String srcSecretKey;
public String getSourceGroup() {
return sourceGroup;
@@ -105,4 +106,28 @@ public class TaskConfig {
public String getOffsetSyncTopic() {
return offsetSyncTopic;
}
+
+ public boolean isSrcAclEnable() {
+ return srcAclEnable;
+ }
+
+ public void setSrcAclEnable(boolean srcAclEnable) {
+ this.srcAclEnable = srcAclEnable;
+ }
+
+ public String getSrcAccessKey() {
+ return srcAccessKey;
+ }
+
+ public void setSrcAccessKey(String srcAccessKey) {
+ this.srcAccessKey = srcAccessKey;
+ }
+
+ public String getSrcSecretKey() {
+ return srcSecretKey;
+ }
+
+ public void setSrcSecretKey(String srcSecretKey) {
+ this.srcSecretKey = srcSecretKey;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
index 01c6787..520c31f 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
@@ -31,7 +31,10 @@ public enum TaskConfigEnum {
TASK_NEXT_POSITION("nextPosition"),
TASK_TOPIC_INFO("taskTopicList"),
TASK_GROUP_INFO("taskGroupList"),
- TASK_SOURCE_RECORD_CONVERTER("source-record-converter");
+ TASK_SOURCE_RECORD_CONVERTER("source-record-converter"),
+ TASK_SOURCE_ACL_ENABLE("srcAclEnable"),
+ TASK_SOURCE_ACCESS_KEY("srcAccessKey"),
+ TASK_SOURCE_SECRET_KEY("srcSecretKey");
private String key;
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
index 07d95c8..b8d6fe5 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
@@ -30,14 +30,23 @@ public class TaskDivideConfig {
private int taskParallelism;
+ private boolean srcAclEnable = false;
+
+ private String srcAccessKey;
+
+ private String srcSecretKey;
+
public TaskDivideConfig(String sourceNamesrvAddr, String srcCluster, String storeTopic, String srcRecordConverter,
- int dataType, int taskParallelism) {
+ int dataType, int taskParallelism, boolean srcAclEnable, String srcAccessKey, String srcSecretKey) {
this.sourceNamesrvAddr = sourceNamesrvAddr;
this.srcCluster = srcCluster;
this.storeTopic = storeTopic;
this.srcRecordConverter = srcRecordConverter;
this.dataType = dataType;
this.taskParallelism = taskParallelism;
+ this.srcAclEnable = srcAclEnable;
+ this.srcAccessKey = srcAccessKey;
+ this.srcSecretKey = srcSecretKey;
}
public String getSourceNamesrvAddr() {
@@ -48,6 +57,14 @@ public class TaskDivideConfig {
this.sourceNamesrvAddr = sourceNamesrvAddr;
}
+ public String getSrcCluster() {
+ return srcCluster;
+ }
+
+ public void setSrcCluster(String srcCluster) {
+ this.srcCluster = srcCluster;
+ }
+
public String getStoreTopic() {
return storeTopic;
}
@@ -79,4 +96,28 @@ public class TaskDivideConfig {
public void setTaskParallelism(int taskParallelism) {
this.taskParallelism = taskParallelism;
}
+
+ public boolean isSrcAclEnable() {
+ return srcAclEnable;
+ }
+
+ public void setSrcAclEnable(boolean srcAclEnable) {
+ this.srcAclEnable = srcAclEnable;
+ }
+
+ public String getSrcAccessKey() {
+ return srcAccessKey;
+ }
+
+ public void setSrcAccessKey(String srcAccessKey) {
+ this.srcAccessKey = srcAccessKey;
+ }
+
+ public String getSrcSecretKey() {
+ return srcSecretKey;
+ }
+
+ public void setSrcSecretKey(String srcSecretKey) {
+ this.srcSecretKey = srcSecretKey;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
index 1f01be0..708a5b0 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
@@ -63,6 +63,9 @@ public class DivideTaskByConsistentHash extends TaskDivideStrategy {
keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
config.add(keyValue);
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
index e587250..bbfa580 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
@@ -54,6 +54,9 @@ public class DivideTaskByQueue extends TaskDivideStrategy {
keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
config.add(keyValue);
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
index 77ea7cc..0d13a5e 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
@@ -49,6 +49,9 @@ public class DivideTaskByTopic extends TaskDivideStrategy {
keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(taskTopicList.get(i)));
keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ACL_ENABLE.getKey(), String.valueOf(tdc.isSrcAclEnable()));
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ACCESS_KEY.getKey(), tdc.getSrcAccessKey());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_SECRET_KEY.getKey(), tdc.getSrcSecretKey());
config.add(keyValue);
}
[rocketmq-connect] 27/39: [ISSUE #478] TopicList is null exception and frequent requestTaskReconfiguration (#483)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit f55d2132317e94529a9e70fbcd896f1e42c51a33
Author: zhoubo <87...@qq.com>
AuthorDate: Thu Dec 19 09:24:00 2019 +0800
[ISSUE #478] TopicList is null exception and frequent requestTaskReconfiguration (#483)
* TopicList is null exception and frequent requestTaskReconfiguration
* https://github.com/apache/rocketmq-externals/issues/478
* * Runtime add some log
* Fix replicator add new topic frequent requestTaskReconfiguration bug
---
.../rocketmq/replicator/RmqSourceReplicator.java | 30 ++++++++++++++--------
.../apache/rocketmq/replicator/RmqSourceTask.java | 2 +-
.../strategy/DivideTaskByConsistentHash.java | 3 ++-
.../replicator/strategy/DivideTaskByQueue.java | 4 ++-
.../replicator/strategy/DivideTaskByTopic.java | 6 ++---
.../replicator/strategy/TaskDivideStrategy.java | 3 ++-
6 files changed, 30 insertions(+), 18 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index e868b89..82744ed 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -63,7 +63,7 @@ public class RmqSourceReplicator extends SourceConnector {
private RmqConnectorConfig replicatorConfig;
- private Map<String, List<TaskTopicInfo>> topicRouteMap;
+ private Map<String, Set<TaskTopicInfo>> topicRouteMap;
private volatile boolean configValid = false;
@@ -75,7 +75,7 @@ public class RmqSourceReplicator extends SourceConnector {
private ScheduledExecutorService executor;
public RmqSourceReplicator() {
- topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
+ topicRouteMap = new HashMap<>();
replicatorConfig = new RmqConnectorConfig();
executor = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("RmqSourceReplicator-SourceWatcher-%d").daemon(true).build());
}
@@ -130,34 +130,42 @@ public class RmqSourceReplicator extends SourceConnector {
@Override
public void start() {
startMQAdminTools();
+ buildRoute();
startListner();
}
public void startListner() {
executor.scheduleAtFixedRate(new Runnable() {
+
+ boolean first = true;
+ Map<String, Set<TaskTopicInfo>> origin = null;
+
+
@Override public void run() {
- Map<String, List<TaskTopicInfo>> origin = topicRouteMap;
- topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
buildRoute();
-
+ if (first) {
+ origin = new HashMap<>(topicRouteMap);
+ first = false;
+ }
if (!compare(origin, topicRouteMap)) {
context.requestTaskReconfiguration();
+ origin = new HashMap<>(topicRouteMap);
}
}
}, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
}
- public boolean compare(Map<String, List<TaskTopicInfo>> origin, Map<String, List<TaskTopicInfo>> updated) {
+ public boolean compare(Map<String, Set<TaskTopicInfo>> origin, Map<String, Set<TaskTopicInfo>> updated) {
if (origin.size() != updated.size()) {
return false;
}
- for (Map.Entry<String, List<TaskTopicInfo>> entry : origin.entrySet()) {
+ for (Map.Entry<String, Set<TaskTopicInfo>> entry : origin.entrySet()) {
if (!updated.containsKey(entry.getKey())) {
return false;
}
- List<TaskTopicInfo> originTasks = entry.getValue();
- List<TaskTopicInfo> updateTasks = updated.get(entry.getKey());
+ Set<TaskTopicInfo> originTasks = entry.getValue();
+ Set<TaskTopicInfo> updateTasks = updated.get(entry.getKey());
if (originTasks.size() != updateTasks.size()) {
return false;
}
@@ -249,7 +257,7 @@ public class RmqSourceReplicator extends SourceConnector {
TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
if (!topicRouteMap.containsKey(topic)) {
- topicRouteMap.put(topic, new ArrayList<TaskTopicInfo>());
+ topicRouteMap.put(topic, new HashSet<>(16));
}
for (QueueData qd : topicRouteData.getQueueDatas()) {
if (brokerNameSet.contains(qd.getBrokerName())) {
@@ -270,7 +278,7 @@ public class RmqSourceReplicator extends SourceConnector {
}
}
- public Map<String, List<TaskTopicInfo>> getTopicRouteMap() {
+ public Map<String, Set<TaskTopicInfo>> getTopicRouteMap() {
return this.topicRouteMap;
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index b504e85..3e8d78b 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -121,7 +121,7 @@ public class RmqSourceTask extends SourceTask {
log.info("RocketMQ source task started");
}
- public void stop() {
+ @Override public void stop() {
if (started) {
if (this.consumer != null) {
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
index b027246..1f01be0 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter;
import org.apache.rocketmq.common.consistenthash.Node;
import org.apache.rocketmq.replicator.config.DataType;
@@ -31,7 +32,7 @@ import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import org.apache.rocketmq.replicator.config.TaskTopicInfo;
public class DivideTaskByConsistentHash extends TaskDivideStrategy {
- @Override public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
+ @Override public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<>();
int parallelism = tdc.getTaskParallelism();
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
index 0a493de..e587250 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
@@ -20,6 +20,7 @@ import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.internal.DefaultKeyValue;
import java.util.HashMap;
+import java.util.Set;
import org.apache.rocketmq.replicator.config.DataType;
import org.apache.rocketmq.replicator.config.TaskConfigEnum;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
@@ -29,7 +30,8 @@ import java.util.Map;
import org.apache.rocketmq.replicator.config.TaskTopicInfo;
public class DivideTaskByQueue extends TaskDivideStrategy {
- public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
+
+ @Override public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
index f8dc8cf..77ea7cc 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
@@ -19,7 +19,7 @@ package org.apache.rocketmq.replicator.strategy;
import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.internal.DefaultKeyValue;
-import org.apache.rocketmq.common.message.MessageQueue;
+import java.util.Set;
import org.apache.rocketmq.replicator.config.*;
import java.util.ArrayList;
import java.util.HashMap;
@@ -28,13 +28,13 @@ import java.util.Map;
public class DivideTaskByTopic extends TaskDivideStrategy {
- public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
+ @Override public List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
int id = -1;
Map<Integer, List<TaskTopicInfo>> taskTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
- for (Map.Entry<String, List<TaskTopicInfo>> entry : topicRouteMap.entrySet()) {
+ for (Map.Entry<String, Set<TaskTopicInfo>> entry : topicRouteMap.entrySet()) {
int ind = ++id % parallelism;
if (!taskTopicList.containsKey(ind)) {
taskTopicList.put(ind, new ArrayList<TaskTopicInfo>());
diff --git a/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java b/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
index 6f58bb3..89ed060 100644
--- a/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.replicator.strategy;
import io.openmessaging.KeyValue;
+import java.util.Set;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import java.util.List;
import java.util.Map;
@@ -24,5 +25,5 @@ import org.apache.rocketmq.replicator.config.TaskTopicInfo;
public abstract class TaskDivideStrategy {
- public abstract List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicMap, TaskDivideConfig tdc);
+ public abstract List<KeyValue> divide(Map<String, Set<TaskTopicInfo>> topicMap, TaskDivideConfig tdc);
}
[rocketmq-connect] 18/39: chore(runtime/replicator): change rocketmq dependency from 4.4.0 to 4.5.2
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit dfb32fd86f7b9990119ccf4ce1346e048ddcfea6
Author: Yongqi Feng <fy...@163.com>
AuthorDate: Sat Oct 19 13:14:13 2019 +0800
chore(runtime/replicator): change rocketmq dependency from 4.4.0 to 4.5.2
1. change rocketmq dependency from 4.4.0 to 4.5.2
2. fix replicator can't load class
3. fix replicator can't start
Closes #431
---
pom.xml | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index b528c80..51a6177 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,9 +24,21 @@
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-replicator</artifactId>
<version>0.1.0-SNAPSHOT</version>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>6</source>
+ <target>6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
<properties>
- <rocketmq.version>4.4.0</rocketmq.version>
+ <rocketmq.version>4.5.2</rocketmq.version>
</properties>
<dependencies>
[rocketmq-connect] 19/39: feat(replicator):add consumer offset sync
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit a3217626bb4c35b5fc71e4a5b704601355a48c06
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Thu Oct 3 23:57:38 2019 +0800
feat(replicator):add consumer offset sync
- sync src cluster consumer groups offset to message store
- Add MetaSourceConnector as sourceConnector for task assign
- Add MetaSourceTask as SourceTask for offset read and message sink
- Add java 1.8 check
Closes #427
---
README.md | 12 ++
pom.xml | 7 +-
.../apache/rocketmq/replicator/MetaSourceTask.java | 163 +++++++++++++++++
.../apache/rocketmq/replicator/RmqConstants.java | 4 +
.../rocketmq/replicator/RmqMetaReplicator.java | 196 +++++++++++++++++++++
.../rocketmq/replicator/RmqSourceReplicator.java | 79 +++------
.../apache/rocketmq/replicator/RmqSourceTask.java | 40 +++--
.../apache/rocketmq/replicator/common/Utils.java | 50 +++++-
.../rocketmq/replicator/config/ConfigDefine.java | 4 +
.../rocketmq/replicator/config/DataType.java | 3 +-
.../replicator/config/RmqConnectorConfig.java | 130 ++++++++++++++
.../rocketmq/replicator/config/TaskConfig.java | 29 +++
.../rocketmq/replicator/config/TaskConfigEnum.java | 3 +
.../replicator/config/TaskDivideConfig.java | 5 +-
.../rocketmq/replicator/offset/OffsetSync.java | 65 +++++++
.../replicator/offset/OffsetSyncStore.java | 90 ++++++++++
.../rocketmq/replicator/schema/FieldName.java | 3 +-
.../replicator/RmqSourceReplicatorTest.java | 7 +-
18 files changed, 809 insertions(+), 81 deletions(-)
diff --git a/README.md b/README.md
index 332db6e..538568b 100644
--- a/README.md
+++ b/README.md
@@ -16,6 +16,18 @@ http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}
http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}/stop
````
+## rocketmq-meta-connector启动
+````
+http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}
+?config={"connector-class":"org.apache.rocketmq.replicator.RmqMetaReplicator","source-rocketmq":"xxxx:9876","target-rocketmq":"xxxxxxx:9876","replicator-store-topic":"replicatorTopic","offset.sync.topic":"syncTopic","taskDivideStrategy":"0","white-list":"TopicTest,TopicTest2","task-parallelism":"2","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+````
+
+
+## rocketmq-rocketmq-connector停止
+````
+http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}/stop
+````
+
## rocketmq-replicator参数说明
parameter | type | must | description | sample value
diff --git a/pom.xml b/pom.xml
index 51a6177..dee5b08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,8 +30,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
- <source>6</source>
- <target>6</target>
+ <source>8</source>
+ <target>8</target>
</configuration>
</plugin>
</plugins>
@@ -107,7 +107,7 @@
<id>release-all</id>
<build>
<plugins>
- <plugin>
+ <plugin>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
@@ -128,6 +128,7 @@
<finalName>rocketmq-replicator-${project.version}</finalName>
</build>
</profile>
+
</profiles>
</project>
diff --git a/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
new file mode 100644
index 0000000..16bf464
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.replicator;
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.data.DataEntryBuilder;
+import io.openmessaging.connector.api.data.EntryType;
+import io.openmessaging.connector.api.data.Field;
+import io.openmessaging.connector.api.data.FieldType;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.SourceDataEntry;
+import io.openmessaging.connector.api.source.SourceTask;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.admin.ConsumeStats;
+import org.apache.rocketmq.common.admin.OffsetWrapper;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.replicator.common.ConstDefine;
+import org.apache.rocketmq.replicator.common.Utils;
+import org.apache.rocketmq.replicator.config.ConfigUtil;
+import org.apache.rocketmq.replicator.config.TaskConfig;
+import org.apache.rocketmq.replicator.offset.OffsetSyncStore;
+import org.apache.rocketmq.replicator.schema.FieldName;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MetaSourceTask extends SourceTask {
+
+ private static final Logger log = LoggerFactory.getLogger(RmqSourceTask.class);
+
+ private final String taskId;
+ private final TaskConfig config;
+ private DefaultMQAdminExt srcMQAdminExt;
+ private volatile boolean started = false;
+
+ private OffsetSyncStore store;
+
+ public MetaSourceTask() {
+ this.config = new TaskConfig();
+ this.taskId = Utils.createTaskId(Thread.currentThread().getName());
+ }
+
+ @Override public void start(KeyValue config) {
+ ConfigUtil.load(config, this.config);
+
+ startAdmin();
+
+ this.store = new OffsetSyncStore(this.srcMQAdminExt, this.config);
+ this.started = true;
+ }
+
+ @Override public void stop() {
+ if (started) {
+ started = false;
+ }
+ }
+
+ @Override public void pause() {
+
+ }
+
+ @Override public void resume() {
+
+ }
+
+ @Override public Collection<SourceDataEntry> poll() {
+ log.debug("polling...");
+ List<String> groups = JSONObject.parseArray(this.config.getTaskGroupList(), String.class);
+
+ if (groups == null) {
+ log.info("no group in task.");
+ try {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(10));
+ } catch (InterruptedException e) {
+ throw new IllegalStateException(e);
+ }
+ return Collections.emptyList();
+ }
+ List<SourceDataEntry> res = new ArrayList<>();
+ for (String group : groups) {
+ ConsumeStats stats;
+ try {
+ stats = this.srcMQAdminExt.examineConsumeStats(group);
+ } catch (Exception e) {
+ log.error("admin get consumer info failed for consumer groups: " + group, e);
+ continue;
+ }
+
+ for (Map.Entry<MessageQueue, OffsetWrapper> offsetTable : stats.getOffsetTable().entrySet()) {
+
+ MessageQueue mq = offsetTable.getKey();
+ long srcOffset = offsetTable.getValue().getConsumerOffset();
+ long targetOffset = this.store.convertTargetOffset(mq, srcOffset);
+
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put(RmqConstants.NEXT_POSITION, srcOffset);
+
+ Schema schema = new Schema();
+ schema.setDataSource(this.config.getSourceRocketmq());
+ schema.setName(mq.getTopic());
+ schema.setFields(new ArrayList<>());
+ schema.getFields().add(new Field(0,
+ FieldName.OFFSET.getKey(), FieldType.INT64));
+
+ DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
+ dataEntryBuilder.timestamp(System.currentTimeMillis())
+ .queue(this.config.getStoreTopic())
+ .entryType(EntryType.UPDATE);
+ dataEntryBuilder.putFiled(FieldName.OFFSET.getKey(), targetOffset);
+
+ SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
+ ByteBuffer.wrap(RmqConstants.getPartition(
+ mq.getTopic(),
+ mq.getBrokerName(),
+ String.valueOf(mq.getQueueId())).getBytes(StandardCharsets.UTF_8)),
+ ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8))
+ );
+ String targetTopic = new StringBuilder().append(group).append("-").append(mq.getTopic())
+ .append("-").append(mq.getQueueId()).toString();
+ sourceDataEntry.setQueueName(targetTopic);
+ res.add(sourceDataEntry);
+ }
+ }
+ return res;
+ }
+
+ private void startAdmin() {
+ this.srcMQAdminExt = new DefaultMQAdminExt();
+ this.srcMQAdminExt.setNamesrvAddr(this.config.getSourceRocketmq());
+ this.srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(this.config.getSourceRocketmq()));
+ try {
+ this.srcMQAdminExt.start();
+ } catch (MQClientException e) {
+ log.error("start src mq admin failed.", e);
+ throw new IllegalStateException("start src mq admin failed");
+ }
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java b/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
index 4994abe..290ab1c 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
@@ -31,4 +31,8 @@ public class RmqConstants {
public static String getPartition(String topic, String broker, String queueId) {
return new StringBuilder().append(broker).append(topic).append(queueId).toString();
}
+
+ public static String getOffsetTag(String topic, String broker, String queueId, String group) {
+ return new StringBuilder().append(broker).append(topic).append(queueId).append(group).toString();
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
new file mode 100644
index 0000000..38e5af2
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.replicator;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import io.openmessaging.connector.api.source.SourceConnector;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.protocol.body.ClusterInfo;
+import org.apache.rocketmq.common.protocol.body.ConsumeStatsList;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.exception.RemotingConnectException;
+import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
+import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
+import org.apache.rocketmq.replicator.common.ConstDefine;
+import org.apache.rocketmq.replicator.common.Utils;
+import org.apache.rocketmq.replicator.config.RmqConnectorConfig;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RmqMetaReplicator extends SourceConnector {
+ private static final Logger log = LoggerFactory.getLogger(RmqSourceReplicator.class);
+
+ private static final Set<String> INNER_CONSUMER_GROUPS = new HashSet<>();
+
+ private RmqConnectorConfig replicatorConfig;
+
+ private volatile boolean configValid = false;
+ private Set<String> knownGroups;
+ private DefaultMQAdminExt srcMQAdminExt;
+ private volatile boolean adminStarted;
+ private ScheduledExecutorService executor;
+
+ static {
+ INNER_CONSUMER_GROUPS.add("TOOLS_CONSUMER");
+ INNER_CONSUMER_GROUPS.add("FILTERSRV_CONSUMER");
+ INNER_CONSUMER_GROUPS.add("__MONITOR_CONSUMER");
+ INNER_CONSUMER_GROUPS.add("CLIENT_INNER_PRODUCER");
+ INNER_CONSUMER_GROUPS.add("SELF_TEST_P_GROUP");
+ INNER_CONSUMER_GROUPS.add("SELF_TEST_C_GROUP");
+ INNER_CONSUMER_GROUPS.add("SELF_TEST_TOPIC");
+ INNER_CONSUMER_GROUPS.add("OFFSET_MOVED_EVENT");
+ INNER_CONSUMER_GROUPS.add("CID_ONS-HTTP-PROXY");
+ INNER_CONSUMER_GROUPS.add("CID_ONSAPI_PERMISSION");
+ INNER_CONSUMER_GROUPS.add("CID_ONSAPI_OWNER");
+ INNER_CONSUMER_GROUPS.add("CID_ONSAPI_PULL");
+ }
+
+ public RmqMetaReplicator() {
+ replicatorConfig = new RmqConnectorConfig();
+ knownGroups = new HashSet<>();
+ executor = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("RmqMetaReplicator-SourceWatcher-%d").daemon(true).build());
+ }
+
+ @Override public String verifyAndSetConfig(KeyValue config) {
+ log.info("verifyAndSetConfig...");
+ try {
+ replicatorConfig.validate(config);
+ } catch (IllegalArgumentException e) {
+ return e.getMessage();
+ }
+
+ this.configValid = true;
+ return "";
+ }
+
+ @Override public void start() {
+ log.info("starting...");
+ startMQAdminTools();
+ executor.scheduleAtFixedRate(() ->
+ {
+ try {
+ refreshConsuemrGroups();
+ } catch (Exception e) {
+ log.error("refresh consumer groups failed.", e);
+ }
+ }, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
+ }
+
+ @Override public void stop() {
+ log.info("stopping...");
+ this.executor.shutdown();
+ }
+
+ @Override public void pause() {
+
+ }
+
+ @Override public void resume() {
+
+ }
+
+ @Override public Class<? extends Task> taskClass() {
+ return MetaSourceTask.class;
+ }
+
+ @Override public List<KeyValue> taskConfigs() {
+ log.debug("preparing taskConfig...");
+ if (!configValid) {
+ return new ArrayList<>();
+ }
+
+ startMQAdminTools();
+
+ try {
+ this.knownGroups = this.fetchConsumerGroups();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return Utils.groupPartitions(new ArrayList<>(this.knownGroups), this.replicatorConfig.getTaskParallelism(), replicatorConfig);
+ }
+
+ private synchronized void startMQAdminTools() {
+ if (!configValid || adminStarted) {
+ return;
+ }
+ RPCHook rpcHook = null;
+ this.srcMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ this.srcMQAdminExt.setNamesrvAddr(this.replicatorConfig.getSrcNamesrvs());
+ this.srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getSrcNamesrvs()));
+
+ try {
+ this.srcMQAdminExt.start();
+ log.info("RocketMQ srcMQAdminExt started");
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `srcMQAdminExt` exception.", e);
+ }
+ adminStarted = true;
+ }
+
+ private void refreshConsuemrGroups() throws InterruptedException, RemotingConnectException, MQBrokerException, RemotingTimeoutException, MQClientException, RemotingSendRequestException {
+ log.debug("refreshConsuemrGroups...");
+ Set<String> groups = fetchConsumerGroups();
+ Set<String> newGroups = new HashSet<>();
+ Set<String> deadGroups = new HashSet<>();
+ newGroups.addAll(groups);
+ newGroups.removeAll(knownGroups);
+ deadGroups.addAll(knownGroups);
+ deadGroups.removeAll(groups);
+ if (!newGroups.isEmpty() || !deadGroups.isEmpty()) {
+ log.info("reconfig consumer groups, new Groups: {} , dead groups: {}, previous groups: {}", newGroups, deadGroups, knownGroups);
+ knownGroups = groups;
+ context.requestTaskReconfiguration();
+ }
+ }
+
+ private Set<String> fetchConsumerGroups() throws InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
+ return listGroups().stream().filter(this::skipInnerGroup).collect(Collectors.toSet());
+ }
+
+ private Set<String> listGroups() throws InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
+ Set<String> groups = new HashSet<>();
+ ClusterInfo clusterInfo = this.srcMQAdminExt.examineBrokerClusterInfo();
+ String[] addrs = clusterInfo.retrieveAllAddrByCluster(this.replicatorConfig.getSrcCluster());
+ for (String addr : addrs) {
+ ConsumeStatsList stats = this.srcMQAdminExt.fetchConsumeStatsInBroker(addr, true, 3 * 1000);
+ stats.getConsumeStatsList().stream().map(kv -> kv.keySet()).forEach(groups::addAll);
+ }
+ return groups;
+ }
+
+ private boolean skipInnerGroup(String group) {
+ if (INNER_CONSUMER_GROUPS.contains(group) || group.startsWith("CID_RMQ_SYS_") || group.startsWith("PositionManage") ||
+ group.startsWith("ConfigManage") || group.startsWith("OffsetManage")) {
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index 429102e..c560840 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -27,7 +27,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -47,12 +46,9 @@ import org.apache.rocketmq.replicator.common.ConstDefine;
import org.apache.rocketmq.replicator.common.Utils;
import org.apache.rocketmq.replicator.config.ConfigDefine;
import org.apache.rocketmq.replicator.config.DataType;
+import org.apache.rocketmq.replicator.config.RmqConnectorConfig;
import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import org.apache.rocketmq.replicator.config.TaskTopicInfo;
-import org.apache.rocketmq.replicator.strategy.DivideStrategyEnum;
-import org.apache.rocketmq.replicator.strategy.DivideTaskByQueue;
-import org.apache.rocketmq.replicator.strategy.DivideTaskByTopic;
-import org.apache.rocketmq.replicator.strategy.TaskDivideStrategy;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,18 +61,12 @@ public class RmqSourceReplicator extends SourceConnector {
private boolean syncRETRY = false;
- private KeyValue replicatorConfig;
+ private RmqConnectorConfig replicatorConfig;
private Map<String, List<TaskTopicInfo>> topicRouteMap;
- private TaskDivideStrategy taskDivideStrategy;
-
- private Set<String> whiteList;
-
private volatile boolean configValid = false;
- private int taskParallelism = 1;
-
private DefaultMQAdminExt srcMQAdminExt;
private DefaultMQAdminExt targetMQAdminExt;
@@ -86,7 +76,7 @@ public class RmqSourceReplicator extends SourceConnector {
public RmqSourceReplicator() {
topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
- whiteList = new HashSet<String>();
+ replicatorConfig = new RmqConnectorConfig();
executor = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("RmqSourceReplicator-SourceWatcher-%d").daemon(true).build());
}
@@ -96,14 +86,14 @@ public class RmqSourceReplicator extends SourceConnector {
}
RPCHook rpcHook = null;
this.srcMQAdminExt = new DefaultMQAdminExt(rpcHook);
- this.srcMQAdminExt.setNamesrvAddr(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ));
+ this.srcMQAdminExt.setNamesrvAddr(this.replicatorConfig.getSrcNamesrvs());
this.srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ)));
+ this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getSrcNamesrvs()));
this.targetMQAdminExt = new DefaultMQAdminExt(rpcHook);
- this.targetMQAdminExt.setNamesrvAddr(this.replicatorConfig.getString(ConfigDefine.CONN_TARGET_RMQ));
+ this.targetMQAdminExt.setNamesrvAddr(this.replicatorConfig.getTargetNamesrvs());
this.targetMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- this.targetMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getString(ConfigDefine.CONN_TARGET_RMQ)));
+ this.targetMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getTargetNamesrvs()));
try {
this.srcMQAdminExt.start();
@@ -128,29 +118,11 @@ public class RmqSourceReplicator extends SourceConnector {
}
}
- // Check the whitelist, whitelist is required.
- String whileListStr = config.getString(ConfigDefine.CONN_WHITE_LIST);
- String[] wl = whileListStr.trim().split(",");
- if (wl.length <= 0)
- return "White list must be not empty.";
- else {
- for (String t : wl) {
- this.whiteList.add(t.trim());
- }
- }
-
- if (config.containsKey(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) &&
- config.getInt(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) == DivideStrategyEnum.BY_QUEUE.ordinal()) {
- this.taskDivideStrategy = new DivideTaskByQueue();
- } else {
- this.taskDivideStrategy = new DivideTaskByTopic();
- }
-
- if (config.containsKey(ConfigDefine.CONN_TASK_PARALLELISM)) {
- this.taskParallelism = config.getInt(ConfigDefine.CONN_TASK_PARALLELISM);
+ try {
+ this.replicatorConfig.validate(config);
+ } catch (IllegalArgumentException e) {
+ return e.getMessage();
}
-
- this.replicatorConfig = config;
this.configValid = true;
return "";
}
@@ -173,7 +145,7 @@ public class RmqSourceReplicator extends SourceConnector {
context.requestTaskReconfiguration();
}
}
- }, 30, 30, TimeUnit.SECONDS);
+ }, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
}
public boolean compare(Map<String, List<TaskTopicInfo>> origin, Map<String, List<TaskTopicInfo>> updated) {
@@ -225,21 +197,22 @@ public class RmqSourceReplicator extends SourceConnector {
buildRoute();
TaskDivideConfig tdc = new TaskDivideConfig(
- this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ),
- this.replicatorConfig.getString(ConfigDefine.CONN_STORE_TOPIC),
- this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER),
+ this.replicatorConfig.getSrcNamesrvs(),
+ this.replicatorConfig.getSrcCluster(),
+ this.replicatorConfig.getStoreTopic(),
+ this.replicatorConfig.getConverter(),
DataType.COMMON_MESSAGE.ordinal(),
- this.taskParallelism
+ this.replicatorConfig.getTaskParallelism()
);
- return this.taskDivideStrategy.divide(this.topicRouteMap, tdc);
+ return this.replicatorConfig.getTaskDivideStrategy().divide(this.topicRouteMap, tdc);
}
public void buildRoute() {
List<Pattern> patterns = new ArrayList<Pattern>();
- String srcCluster = this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_CLUSTER);
+ String srcCluster = this.replicatorConfig.getSrcCluster();
try {
Set<String> targetTopicSet = fetchTargetTopics();
- for (String topic : this.whiteList) {
+ for (String topic : this.replicatorConfig.getWhiteList()) {
Pattern pattern = Pattern.compile(topic);
patterns.add(pattern);
}
@@ -290,16 +263,12 @@ public class RmqSourceReplicator extends SourceConnector {
}
}
- public void setWhiteList(Set<String> whiteList) {
- this.whiteList = whiteList;
- }
-
public Map<String, List<TaskTopicInfo>> getTopicRouteMap() {
return this.topicRouteMap;
}
public Set<String> fetchTargetTopics() throws RemotingException, MQClientException, InterruptedException {
- String targetCluster = this.replicatorConfig.getString(ConfigDefine.CONN_TARGET_CLUSTER);
+ String targetCluster = this.replicatorConfig.getTargetCluster();
TopicList targetTopics = this.targetMQAdminExt.fetchTopicsByCLuster(targetCluster);
return targetTopics.getTopicList();
}
@@ -317,8 +286,8 @@ public class RmqSourceReplicator extends SourceConnector {
*/
public void ensureTargetTopic(String srcTopic,
String targetTopic) throws RemotingException, MQClientException, InterruptedException {
- String srcCluster = this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_CLUSTER);
- String targetCluster = this.replicatorConfig.getString(ConfigDefine.CONN_TARGET_CLUSTER);
+ String srcCluster = this.replicatorConfig.getSrcCluster();
+ String targetCluster = this.replicatorConfig.getTargetCluster();
List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, srcTopic, srcCluster);
if (brokerList.size() == 0) {
@@ -334,7 +303,7 @@ public class RmqSourceReplicator extends SourceConnector {
}
public String generateTargetTopic(String topic) {
- String fmt = this.replicatorConfig.getString(ConfigDefine.CONN_TOPIC_RENAME_FMT);
+ String fmt = this.replicatorConfig.getRenamePattern();
if (StringUtils.isNotEmpty(fmt)) {
Map<String, String> params = new HashMap<String, String>();
params.put("topic", topic);
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index 42dbab8..d965898 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -19,8 +19,20 @@ package org.apache.rocketmq.replicator;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
-import io.openmessaging.connector.api.data.*;
+import io.openmessaging.connector.api.data.DataEntryBuilder;
+import io.openmessaging.connector.api.data.EntryType;
+import io.openmessaging.connector.api.data.Field;
+import io.openmessaging.connector.api.data.FieldType;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.SourceDataEntry;
import io.openmessaging.connector.api.source.SourceTask;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.common.message.MessageExt;
@@ -31,11 +43,11 @@ import org.apache.rocketmq.replicator.config.DataType;
import org.apache.rocketmq.replicator.config.TaskConfig;
import org.apache.rocketmq.replicator.config.TaskTopicInfo;
import org.apache.rocketmq.replicator.schema.FieldName;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
-import java.util.*;
public class RmqSourceTask extends SourceTask {
@@ -52,7 +64,7 @@ public class RmqSourceTask extends SourceTask {
this.config = new TaskConfig();
this.consumer = new DefaultMQPullConsumer();
this.taskId = Utils.createTaskId(Thread.currentThread().getName());
- mqOffsetMap = new HashMap<TaskTopicInfo, Long>();
+ mqOffsetMap = new HashMap<>();
}
public Collection<SourceDataEntry> poll() {
@@ -84,15 +96,15 @@ public class RmqSourceTask extends SourceTask {
for (TaskTopicInfo tti : topicList) {
Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(tti.getTopic());
for (MessageQueue mq : mqs) {
- if (Integer.valueOf(tti.getQueueId()) == mq.getQueueId()) {
+ if (tti.getQueueId() == mq.getQueueId()) {
ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
ByteBuffer.wrap(RmqConstants.getPartition(
mq.getTopic(),
mq.getBrokerName(),
- String.valueOf(mq.getQueueId())).getBytes("UTF-8")));
+ String.valueOf(mq.getQueueId())).getBytes(StandardCharsets.UTF_8)));
if (null != positionInfo && positionInfo.array().length > 0) {
- String positionJson = new String(positionInfo.array(), "UTF-8");
+ String positionJson = new String(positionInfo.array(), StandardCharsets.UTF_8);
JSONObject jsonObject = JSONObject.parseObject(positionJson);
this.config.setNextPosition(jsonObject.getLong(RmqConstants.NEXT_POSITION));
} else {
@@ -129,7 +141,7 @@ public class RmqSourceTask extends SourceTask {
private Collection<SourceDataEntry> pollCommonMessage() {
- List<SourceDataEntry> res = new ArrayList<SourceDataEntry>();
+ List<SourceDataEntry> res = new ArrayList<>();
if (started) {
try {
for (TaskTopicInfo taskTopicConfig : this.mqOffsetMap.keySet()) {
@@ -144,7 +156,7 @@ public class RmqSourceTask extends SourceTask {
Schema schema = new Schema();
schema.setDataSource(this.config.getSourceRocketmq());
schema.setName(taskTopicConfig.getTopic());
- schema.setFields(new ArrayList<Field>());
+ schema.setFields(new ArrayList<>());
schema.getFields().add(new Field(0,
FieldName.COMMON_MESSAGE.getKey(), FieldType.STRING));
@@ -152,12 +164,13 @@ public class RmqSourceTask extends SourceTask {
dataEntryBuilder.timestamp(System.currentTimeMillis())
.queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), JSONObject.toJSONString(msgs));
+
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
ByteBuffer.wrap(RmqConstants.getPartition(
taskTopicConfig.getTopic(),
taskTopicConfig.getBrokerName(),
- String.valueOf(taskTopicConfig.getQueueId())).getBytes("UTF-8")),
- ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8"))
+ String.valueOf(taskTopicConfig.getQueueId())).getBytes(StandardCharsets.UTF_8)),
+ ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8))
);
sourceDataEntry.setQueueName(taskTopicConfig.getTargetTopic());
res.add(sourceDataEntry);
@@ -179,15 +192,16 @@ public class RmqSourceTask extends SourceTask {
}
private Collection<SourceDataEntry> pollTopicConfig() {
- return new ArrayList<SourceDataEntry>();
+ DefaultMQAdminExt srcMQAdminExt;
+ return new ArrayList<>();
}
private Collection<SourceDataEntry> pollBrokerConfig() {
- return new ArrayList<SourceDataEntry>();
+ return new ArrayList<>();
}
private Collection<SourceDataEntry> pollSubConfig() {
- return new ArrayList<SourceDataEntry>();
+ return new ArrayList<>();
}
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index f3e6be6..4134a2a 100644
--- a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
@@ -16,6 +16,9 @@
*/
package org.apache.rocketmq.replicator.common;
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.internal.DefaultKeyValue;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -26,6 +29,9 @@ import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.replicator.config.DataType;
+import org.apache.rocketmq.replicator.config.RmqConnectorConfig;
+import org.apache.rocketmq.replicator.config.TaskConfigEnum;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.apache.rocketmq.tools.command.CommandUtil;
import org.slf4j.Logger;
@@ -48,7 +54,7 @@ public class Utils {
public static String createInstanceName(String namesrvAddr) {
String[] namesrvArray = namesrvAddr.split(";");
- List<String> namesrvList = new ArrayList<String>();
+ List<String> namesrvList = new ArrayList<>();
for (String ns : namesrvArray) {
if (!namesrvList.contains(ns)) {
namesrvList.add(ns);
@@ -60,7 +66,7 @@ public class Utils {
public static List<BrokerData> examineBrokerData(DefaultMQAdminExt defaultMQAdminExt, String topic,
String cluster) throws RemotingException, MQClientException, InterruptedException {
- List<BrokerData> brokerList = new ArrayList<BrokerData>();
+ List<BrokerData> brokerList = new ArrayList<>();
TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
if (topicRouteData.getBrokerDatas() != null) {
@@ -79,7 +85,7 @@ public class Utils {
CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName);
for (String addr : masterSet) {
defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig);
- log.info("create topic to %s success.%n", addr);
+ log.info("create topic to {} success.%n", addr);
}
if (topicConfig.isOrder()) {
@@ -94,12 +100,44 @@ public class Utils {
}
defaultMQAdminExt.createOrUpdateOrderConf(topicConfig.getTopicName(),
orderConf.toString(), true);
- log.info("set cluster orderConf=[%s]", orderConf);
+ log.info("set cluster orderConf=[{}]", orderConf);
}
-
- return;
} catch (Exception e) {
throw new IllegalArgumentException("create topic: " + topicConfig + "failed", e);
}
}
+
+ public static List<KeyValue> groupPartitions(List<String> elements, int numGroups, RmqConnectorConfig tdc) {
+ if (numGroups <= 0)
+ throw new IllegalArgumentException("Number of groups must be positive.");
+
+ List<KeyValue> result = new ArrayList<>(numGroups);
+
+ // Each group has either n+1 or n raw partitions
+ int perGroup = elements.size() / numGroups;
+ int leftover = elements.size() - (numGroups * perGroup);
+
+ int assigned = 0;
+ for (int group = 0; group < numGroups; group++) {
+ int numThisGroup = group < leftover ? perGroup + 1 : perGroup;
+ KeyValue keyValue = new DefaultKeyValue();
+ List<String> groupList = new ArrayList<>();
+ for (int i = 0; i < numThisGroup; i++) {
+ groupList.add(elements.get(assigned));
+ assigned++;
+ }
+ keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSrcNamesrvs());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_CLUSTER.getKey(), tdc.getSrcCluster());
+ keyValue.put(TaskConfigEnum.TASK_OFFSET_SYNC_TOPIC.getKey(), tdc.getSrcCluster());
+ keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.OFFSET.ordinal());
+ keyValue.put(TaskConfigEnum.TASK_GROUP_INFO.getKey(), JSONObject.toJSONString(groupList));
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getConverter());
+ result.add(keyValue);
+
+ log.debug("allocate group partition: {}", keyValue);
+ }
+
+ return result;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
index 5b15821..a3514ee 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
@@ -43,6 +43,10 @@ public class ConfigDefine {
public static final String CONN_TOPIC_RENAME_FMT = "topic.rename.format";
+ public static final String REFRESH_INTERVAL = "refresh.interval";
+
+ public static final String OFFSET_SYNC_TOPIC = "offset.sync.topic";
+
/**
* The required key for all configurations.
*/
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/DataType.java b/src/main/java/org/apache/rocketmq/replicator/config/DataType.java
index 75f772e..60dc330 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/DataType.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/DataType.java
@@ -21,5 +21,6 @@ public enum DataType {
COMMON_MESSAGE,
TOPIC_CONFIG,
BROKER_CONFIG,
- SUB_CONFIG
+ SUB_CONFIG,
+ OFFSET
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java b/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
new file mode 100644
index 0000000..395d75d
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.replicator.config;
+
+import io.openmessaging.KeyValue;
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.rocketmq.replicator.strategy.DivideStrategyEnum;
+import org.apache.rocketmq.replicator.strategy.DivideTaskByQueue;
+import org.apache.rocketmq.replicator.strategy.DivideTaskByTopic;
+import org.apache.rocketmq.replicator.strategy.TaskDivideStrategy;
+
+public class RmqConnectorConfig {
+
+ private int taskParallelism;
+ private Set<String> whiteList;
+ private String srcNamesrvs;
+ private String targetNamesrvs;
+ private String srcCluster;
+ private String targetCluster;
+ private TaskDivideStrategy taskDivideStrategy;
+ private String storeTopic;
+ private String converter;
+ private long refreshInterval;
+ private String renamePattern;
+ private String offsetSyncTopic;
+
+ public RmqConnectorConfig() {
+ }
+
+ public void validate(KeyValue config) {
+ this.taskParallelism = config.getInt(ConfigDefine.CONN_TASK_PARALLELISM, 1);
+
+ int strategy = config.getInt(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY, DivideStrategyEnum.BY_QUEUE.ordinal());
+ if (strategy == DivideStrategyEnum.BY_QUEUE.ordinal()) {
+ this.taskDivideStrategy = new DivideTaskByQueue();
+ } else {
+ this.taskDivideStrategy = new DivideTaskByTopic();
+ }
+
+ buildWhiteList(config);
+
+ srcNamesrvs = config.getString(ConfigDefine.CONN_SOURCE_RMQ);
+ srcCluster = config.getString(ConfigDefine.CONN_SOURCE_CLUSTER);
+ targetNamesrvs = config.getString(ConfigDefine.CONN_TARGET_RMQ);
+ targetCluster = config.getString(ConfigDefine.CONN_TARGET_CLUSTER);
+
+ storeTopic = config.getString(ConfigDefine.CONN_STORE_TOPIC);
+ converter = config.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER);
+ refreshInterval = config.getLong(ConfigDefine.REFRESH_INTERVAL, 3);
+ renamePattern = config.getString(ConfigDefine.CONN_TOPIC_RENAME_FMT);
+ offsetSyncTopic = config.getString(ConfigDefine.OFFSET_SYNC_TOPIC);
+ }
+
+ private void buildWhiteList(KeyValue config) {
+ this.whiteList = new HashSet<String>();
+ String whileListStr = config.getString(ConfigDefine.CONN_WHITE_LIST, "");
+ String[] wl = whileListStr.trim().split(",");
+ if (wl.length <= 0)
+ throw new IllegalArgumentException("White list must be not empty.");
+ else {
+ this.whiteList.clear();
+ for (String t : wl) {
+ this.whiteList.add(t.trim());
+ }
+ }
+ }
+
+ public int getTaskParallelism() {
+ return this.taskParallelism;
+ }
+
+ public Set<String> getWhiteList() {
+ return this.whiteList;
+ }
+
+ public String getSrcNamesrvs() {
+ return this.srcNamesrvs;
+ }
+
+ public String getTargetNamesrvs() {
+ return this.targetNamesrvs;
+ }
+
+ public String getSrcCluster() {
+ return this.srcCluster;
+ }
+
+ public String getTargetCluster() {
+ return this.targetCluster;
+ }
+
+ public TaskDivideStrategy getTaskDivideStrategy() {
+ return this.taskDivideStrategy;
+ }
+
+ public String getStoreTopic() {
+ return this.storeTopic;
+ }
+
+ public String getConverter() {
+ return this.converter;
+ }
+
+ public long getRefreshInterval() {
+ return this.refreshInterval;
+ }
+
+ public String getRenamePattern() {
+ return this.renamePattern;
+ }
+
+ public String getOffsetSyncTopic() {
+ return this.offsetSyncTopic;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
index e85280f..15676ce 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
@@ -16,14 +16,19 @@
*/
package org.apache.rocketmq.replicator.config;
+import java.util.List;
+
public class TaskConfig {
+ private String sourceCluster;
private String storeTopic;
private String sourceGroup;
private String sourceRocketmq;
private Integer dataType;
private Long nextPosition;
private String taskTopicList;
+ private String taskGroupList;
+ private String offsetSyncTopic;
public String getSourceGroup() {
return sourceGroup;
@@ -73,7 +78,31 @@ public class TaskConfig {
return taskTopicList;
}
+ public void setTaskGroupList(String taskGroupList) {
+ this.taskGroupList = taskGroupList;
+ }
+
+ public String getTaskGroupList() {
+ return this.taskGroupList;
+ }
+
public void setTaskTopicList(String taskTopicList) {
this.taskTopicList = taskTopicList;
}
+
+ public void setSourceCluster(String sourceCluster) {
+ this.sourceCluster = sourceCluster;
+ }
+
+ public String getSourceCluster() {
+ return this.sourceCluster;
+ }
+
+ public void setOffsetSyncTopic(String offsetSyncTopic) {
+ this.offsetSyncTopic = offsetSyncTopic;
+ }
+
+ public String getOffsetSyncTopic() {
+ return offsetSyncTopic;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
index 1516f7a..01c6787 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
@@ -21,6 +21,8 @@ public enum TaskConfigEnum {
TASK_ID("taskId"),
TASK_SOURCE_GROUP("sourceGroup"),
TASK_SOURCE_ROCKETMQ("sourceRocketmq"),
+ TASK_SOURCE_CLUSTER("sourceCluster"),
+ TASK_OFFSET_SYNC_TOPIC("offsetSyncTopic"),
TASK_SOURCE_TOPIC("sourceTopic"),
TASK_STORE_ROCKETMQ("storeTopic"),
TASK_DATA_TYPE("dataType"),
@@ -28,6 +30,7 @@ public enum TaskConfigEnum {
TASK_QUEUE_ID("queueId"),
TASK_NEXT_POSITION("nextPosition"),
TASK_TOPIC_INFO("taskTopicList"),
+ TASK_GROUP_INFO("taskGroupList"),
TASK_SOURCE_RECORD_CONVERTER("source-record-converter");
private String key;
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
index e6a8144..07d95c8 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
@@ -20,6 +20,8 @@ public class TaskDivideConfig {
private String sourceNamesrvAddr;
+ private String srcCluster;
+
private String storeTopic;
private String srcRecordConverter;
@@ -28,9 +30,10 @@ public class TaskDivideConfig {
private int taskParallelism;
- public TaskDivideConfig(String sourceNamesrvAddr, String storeTopic, String srcRecordConverter,
+ public TaskDivideConfig(String sourceNamesrvAddr, String srcCluster, String storeTopic, String srcRecordConverter,
int dataType, int taskParallelism) {
this.sourceNamesrvAddr = sourceNamesrvAddr;
+ this.srcCluster = srcCluster;
this.storeTopic = storeTopic;
this.srcRecordConverter = srcRecordConverter;
this.dataType = dataType;
diff --git a/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java b/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
new file mode 100644
index 0000000..4678d5b
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
@@ -0,0 +1,65 @@
+package org.apache.rocketmq.replicator.offset;/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.alibaba.fastjson.JSON;
+import org.apache.rocketmq.common.message.MessageQueue;
+
+public class OffsetSync {
+
+ private MessageQueue mq;
+ private long srcOffset;
+ private long targtOffset;
+
+ public OffsetSync(MessageQueue mq, long srcOffset, long targtOffset) {
+ this.mq = mq;
+ this.srcOffset = srcOffset;
+ this.targtOffset = targtOffset;
+ }
+
+ public void setMq(MessageQueue mq) {
+ this.mq = mq;
+ }
+
+ public void setSrcOffset(long srcOffset) {
+ this.srcOffset = srcOffset;
+ }
+
+ public void setTargtOffset(long targtOffset) {
+ this.targtOffset = targtOffset;
+ }
+
+ public long getSrcOffset() {
+ return this.srcOffset;
+ }
+
+ public long getTargtOffset() {
+ return this.targtOffset;
+ }
+
+ public MessageQueue getMq() {
+ return this.mq;
+ }
+
+ public byte[] encode() {
+ return JSON.toJSONBytes(this);
+ }
+
+ public static OffsetSync decode(byte[] body) {
+ OffsetSync sync = JSON.parseObject(body, OffsetSync.class);
+ return sync;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java b/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
new file mode 100644
index 0000000..5703f39
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
@@ -0,0 +1,90 @@
+package org.apache.rocketmq.replicator.offset;/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
+import org.apache.rocketmq.client.consumer.PullResult;
+import org.apache.rocketmq.client.consumer.PullStatus;
+import org.apache.rocketmq.client.exception.MQBrokerException;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageExt;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.replicator.config.TaskConfig;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+
+public class OffsetSyncStore {
+
+ private DefaultMQAdminExt adminExt;
+ private TaskConfig taskConfig;
+
+ private DefaultMQPullConsumer consumer;
+ private Map<MessageQueue, OffsetSync> syncs;
+ private long lastOffset;
+
+ public OffsetSyncStore(DefaultMQAdminExt adminExt,
+ TaskConfig taskConfig) {
+ this.adminExt = adminExt;
+ this.taskConfig = taskConfig;
+ this.syncs = new HashMap<MessageQueue, OffsetSync>();
+ this.consumer = new DefaultMQPullConsumer();
+ }
+
+ public long convertTargetOffset(MessageQueue mq, long srcOffset) {
+ OffsetSync offsetSync = latestOffsetSync(mq);
+ if (offsetSync.getSrcOffset() > srcOffset) {
+ return -1;
+ }
+ long delta = srcOffset - offsetSync.getSrcOffset();
+ return offsetSync.getTargtOffset() + delta;
+ }
+
+ private boolean sync(
+ Duration pullTimeout) throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
+ TopicRouteData route = adminExt.examineTopicRouteInfo(taskConfig.getOffsetSyncTopic());
+ String brokerName = route.getQueueDatas().get(0).getBrokerName();
+ MessageQueue mq = new MessageQueue(taskConfig.getOffsetSyncTopic(), brokerName, 0);
+
+ PullResult pr = consumer.pull(mq, "", lastOffset, 0, pullTimeout.getNano() / Duration.ofMillis(1).getNano());
+ if (pr.getPullStatus() != PullStatus.FOUND) {
+ return false;
+ }
+ handle(pr);
+ return true;
+ }
+
+ private void handle(PullResult result) {
+ for (MessageExt msg : result.getMsgFoundList()) {
+ byte[] body = msg.getBody();
+ OffsetSync sync = OffsetSync.decode(body);
+ syncs.put(sync.getMq(), sync);
+ }
+ }
+
+ private OffsetSync latestOffsetSync(MessageQueue queue) {
+ return syncs.computeIfAbsent(queue, new Function<MessageQueue, OffsetSync>() {
+ @Override public OffsetSync apply(MessageQueue queue) {
+ return new OffsetSync(queue, -1, -1);
+ }
+ });
+ }
+
+}
diff --git a/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java b/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
index 913ffca..a8acd08 100644
--- a/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
+++ b/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
@@ -17,7 +17,8 @@
package org.apache.rocketmq.replicator.schema;
public enum FieldName {
- COMMON_MESSAGE("MessageExt");
+ COMMON_MESSAGE("MessageExt"),
+ OFFSET("Offset");
private String key;
diff --git a/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java b/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
index f271f14..795c386 100644
--- a/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
+++ b/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
@@ -28,6 +28,7 @@ import org.apache.rocketmq.common.protocol.body.TopicList;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.replicator.config.RmqConnectorConfig;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -51,10 +52,14 @@ public class RmqSourceReplicatorTest {
@Test
public void testGenerateTopic() throws NoSuchFieldException {
RmqSourceReplicator rmqSourceReplicator = Mockito.spy(RmqSourceReplicator.class);
+
+ RmqConnectorConfig config = new RmqConnectorConfig();
KeyValue kv = new DefaultKeyValue();
kv.put(ConfigDefine.CONN_TOPIC_RENAME_FMT, "${topic}.replica");
+ config.validate(kv);
+
Field field = RmqSourceReplicator.class.getDeclaredField("replicatorConfig");
- FieldSetter.setField(rmqSourceReplicator, field, kv);
+ FieldSetter.setField(rmqSourceReplicator, field, config);
String dstTopic = rmqSourceReplicator.generateTargetTopic("dest");
assertThat(dstTopic).isEqualTo("dest.replica");
}
[rocketmq-connect] 07/39: [ISSUE 368]Polish rocketmq replicator implementation (#366)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 40b9fc8d9c4e6b04f06626948c1197be33c60e5b
Author: Heng Du <du...@apache.org>
AuthorDate: Tue Aug 13 14:26:40 2019 +0800
[ISSUE 368]Polish rocketmq replicator implementation (#366)
* Polish rocketmq replicator implementation
* Add documation of rocketmq-replicator
---
README.md | 18 ++-
package.xml | 41 +++++++
pom.xml | 30 ++++-
.../rocketmq/replicator/RmqSourceReplicator.java | 133 +++++++++++----------
.../apache/rocketmq/replicator/RmqSourceTask.java | 5 +-
.../apache/rocketmq/replicator/common/Utils.java | 6 +-
6 files changed, 167 insertions(+), 66 deletions(-)
diff --git a/README.md b/README.md
index 77bcb94..b2c98d1 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,22 @@
# rocketmq-replicator
-# boot-parameter
+## rocketmq-replicator打包
+````
+mvn clean install -Prelease-all -DskipTest -U
+````
+## rocketmq-replicator启动
+````
+http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}
+?config={"connector-class":"org.apache.rocketmq.connector.RmqSourceConnector","source-rocketmq":"xxxx:9876","target-rocketmq":"xxxxxxx:9876","replicator-store-topic":"replicatorTopic","taskDivideStrategy":"0","white-list”:"TopicTest,TopicTest2","task-parallelism":"2","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+````
+
+
+## rocketmq-replicator停止
+````
+http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}/stop
+````
+
+## rocketmq-replicator参数说明
parameter | type | must | description | sample value
---|---|---|---|---|
diff --git a/package.xml b/package.xml
new file mode 100644
index 0000000..5dc18b9
--- /dev/null
+++ b/package.xml
@@ -0,0 +1,41 @@
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2
+ http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <!-- Assembles a packaged version targeting OS installation. -->
+ <id>package</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <includes>
+ <include>README*</include>
+ <include>LICENSE*</include>
+ <include>NOTICE*</include>
+ <include>licenses/</include>
+ <include>notices/</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/conf</directory>
+ <outputDirectory>conf/</outputDirectory>
+ <includes>
+ <include>*</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib/</outputDirectory>
+ <useProjectArtifact>true</useProjectArtifact>
+ <useTransitiveFiltering>true</useTransitiveFiltering>
+ <excludes>
+ <exclude>io.openmessaging:openmessaging-connector</exclude>
+ <exclude>com.alibaba:fastjson</exclude>
+ </excludes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
diff --git a/pom.xml b/pom.xml
index d283ca9..963222e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,8 +74,36 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
- <version>1.2.51</version>
+ <version>1.2.58</version>
</dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>release-all</id>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <descriptors>
+ <descriptor>package.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <finalName>rocketmq-replicator-${project.version}</finalName>
+ </build>
+ </profile>
+ </profiles>
+
</project>
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index c1f350f..12473ab 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -19,11 +19,18 @@ package org.apache.rocketmq.replicator;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.source.SourceConnector;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.replicator.common.ConstDefine;
import org.apache.rocketmq.replicator.common.Utils;
import org.apache.rocketmq.replicator.config.ConfigDefine;
@@ -33,13 +40,10 @@ import org.apache.rocketmq.replicator.strategy.DivideStrategyEnum;
import org.apache.rocketmq.replicator.strategy.DivideTaskByQueue;
import org.apache.rocketmq.replicator.strategy.DivideTaskByTopic;
import org.apache.rocketmq.replicator.strategy.TaskDivideStrategy;
-import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
-
public class RmqSourceReplicator extends SourceConnector {
private static final Logger log = LoggerFactory.getLogger(RmqSourceReplicator.class);
@@ -60,33 +64,56 @@ public class RmqSourceReplicator extends SourceConnector {
private int taskParallelism = 1;
- public RmqSourceReplicator() {
+ private DefaultMQAdminExt defaultMQAdminExt;
+
+ private volatile boolean adminStarted;
+ public RmqSourceReplicator() {
topicRouteMap = new HashMap<String, List<MessageQueue>>();
whiteList = new HashSet<String>();
}
+ private synchronized void startMQAdminTools() {
+ if (!configValid || adminStarted) {
+ return;
+ }
+ RPCHook rpcHook = null;
+ this.defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ this.defaultMQAdminExt.setNamesrvAddr(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ));
+ this.defaultMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ this.defaultMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ)));
+ try {
+ defaultMQAdminExt.start();
+ log.info("RocketMQ defaultMQAdminExt started");
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `defaultMQAdminExt` exception.", e);
+ }
+ adminStarted = true;
+ }
+
+ @Override
public String verifyAndSetConfig(KeyValue config) {
- // check the need key.
- for(String requestKey : ConfigDefine.REQUEST_CONFIG){
- if(!config.containsKey(requestKey)){
+ // Check the need key.
+ for (String requestKey : ConfigDefine.REQUEST_CONFIG) {
+ if (!config.containsKey(requestKey)) {
return "Request config key: " + requestKey;
}
}
- // check the whitelist, whitelist is required.
+ // Check the whitelist, whitelist is required.
String whileListStr = config.getString(ConfigDefine.CONN_WHITE_LIST);
String[] wl = whileListStr.trim().split(",");
- if (wl.length <= 0) return "White list must be not empty.";
+ if (wl.length <= 0)
+ return "White list must be not empty.";
else {
- for (String t: wl) {
+ for (String t : wl) {
this.whiteList.add(t.trim());
}
}
if (config.containsKey(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) &&
- config.getInt(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) == DivideStrategyEnum.BY_QUEUE.ordinal()) {
+ config.getInt(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) == DivideStrategyEnum.BY_QUEUE.ordinal()) {
this.taskDivideStrategy = new DivideTaskByQueue();
} else {
this.taskDivideStrategy = new DivideTaskByTopic();
@@ -101,7 +128,9 @@ public class RmqSourceReplicator extends SourceConnector {
return "";
}
+ @Override
public void start() {
+ startMQAdminTools();
}
public void stop() {
@@ -116,65 +145,49 @@ public class RmqSourceReplicator extends SourceConnector {
}
public Class<? extends Task> taskClass() {
-
+
return RmqSourceTask.class;
}
public List<KeyValue> taskConfigs() {
-
- if (configValid) {
-
- boolean adminStarted = false;
- RPCHook rpcHook = null;
- DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
- defaultMQAdminExt.setNamesrvAddr(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ));
- defaultMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- defaultMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ)));
-
- try {
- defaultMQAdminExt.start();
- adminStarted = true;
- } catch (MQClientException e) {
- log.error("Replicator start failed for `defaultMQAdminExt` exception.", e);
- }
+ if (!configValid) {
+ return new ArrayList<KeyValue>();
+ }
+
+ startMQAdminTools();
+
+ try {
+ for (String topic : this.whiteList) {
+ if ((syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) ||
+ (syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) ||
+ !topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
- if (adminStarted) {
- try {
- for (String topic : this.whiteList) {
- if ((syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) ||
- (syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) ||
- !topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
-
- TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
- if (!topicRouteMap.containsKey(topic)) {
- topicRouteMap.put(topic, new ArrayList<MessageQueue>());
- }
- for (QueueData qd : topicRouteData.getQueueDatas()) {
- for (int i = 0; i < qd.getReadQueueNums(); i++) {
- MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
- topicRouteMap.get(topic).add(mq);
- }
- }
+ TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+ if (!topicRouteMap.containsKey(topic)) {
+ topicRouteMap.put(topic, new ArrayList<MessageQueue>());
+ }
+ for (QueueData qd : topicRouteData.getQueueDatas()) {
+ for (int i = 0; i < qd.getReadQueueNums(); i++) {
+ MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
+ topicRouteMap.get(topic).add(mq);
}
}
- } catch (Exception e) {
- log.error("Fetch topic list error.", e);
- } finally {
- defaultMQAdminExt.shutdown();
}
}
-
- TaskDivideConfig tdc = new TaskDivideConfig(
- this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ),
- this.replicatorConfig.getString(ConfigDefine.CONN_STORE_TOPIC),
- this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER),
- DataType.COMMON_MESSAGE.ordinal(),
- this.taskParallelism
- );
- return this.taskDivideStrategy.divide(this.topicRouteMap, tdc);
- } else {
- return new ArrayList<KeyValue>();
+ } catch (Exception e) {
+ log.error("Fetch topic list error.", e);
+ } finally {
+ defaultMQAdminExt.shutdown();
}
+
+ TaskDivideConfig tdc = new TaskDivideConfig(
+ this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ),
+ this.replicatorConfig.getString(ConfigDefine.CONN_STORE_TOPIC),
+ this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER),
+ DataType.COMMON_MESSAGE.ordinal(),
+ this.taskParallelism
+ );
+ return this.taskDivideStrategy.divide(this.topicRouteMap, tdc);
}
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index 50a88bb..8c8e434 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -122,6 +122,7 @@ public class RmqSourceTask extends SourceTask {
} catch (Exception e) {
log.error("Consumer of task {} start failed.", this.taskId, e);
}
+ log.info("RocketMQ source task started");
}
public void stop() {
@@ -185,7 +186,9 @@ public class RmqSourceTask extends SourceTask {
log.error("Rocketmq replicator task poll error, current config: {}", JSON.toJSONString(config), e);
}
} else {
- log.warn("Rocketmq replicator task is not started.");
+ if (System.currentTimeMillis() % 1000 == 0) {
+ log.warn("Rocketmq replicator task is not started.");
+ }
}
return res;
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index 0e4766d..6888038 100644
--- a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
@@ -23,15 +23,15 @@ import java.util.List;
public class Utils {
public static String createGroupName(String prefix) {
- return new StringBuilder().append(prefix).append("@").append(System.currentTimeMillis()).toString();
+ return new StringBuilder().append(prefix).append("-").append(System.currentTimeMillis()).toString();
}
public static String createGroupName(String prefix, String postfix) {
- return new StringBuilder().append(prefix).append("@").append(postfix).toString();
+ return new StringBuilder().append(prefix).append("-").append(postfix).toString();
}
public static String createTaskId(String prefix) {
- return new StringBuilder().append(prefix).append("@").append(System.currentTimeMillis()).toString();
+ return new StringBuilder().append(prefix).append("-").append(System.currentTimeMillis()).toString();
}
public static String createInstanceName(String namesrvAddr) {
[rocketmq-connect] 38/39: [rocketmq-replicator] Support subGroup whitelist (#843)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 55631bb1fe4d3864c2d45dc7fee9e49764d7b3ac
Author: Git_Yang <zh...@xiaomi.com>
AuthorDate: Fri Oct 29 13:59:17 2021 +0800
[rocketmq-replicator] Support subGroup whitelist (#843)
Signed-off-by: zhangyang21 <zh...@xiaomi.com>
---
.../rocketmq/replicator/RmqMetaReplicator.java | 63 ++++++++++++++--------
1 file changed, 41 insertions(+), 22 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
index e645b6c..bbf3527 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
@@ -29,9 +29,10 @@ import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
-import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.protocol.body.ClusterInfo;
@@ -62,6 +63,7 @@ public class RmqMetaReplicator extends SourceConnector {
private DefaultMQAdminExt targetMQAdminExt;
private volatile boolean adminStarted;
private ScheduledExecutorService executor;
+ private List<Pattern> whiteListPatterns;
static {
INNER_CONSUMER_GROUPS.add("TOOLS_CONSUMER");
@@ -81,6 +83,7 @@ public class RmqMetaReplicator extends SourceConnector {
public RmqMetaReplicator() {
replicatorConfig = new RmqConnectorConfig();
knownGroups = new HashSet<>();
+ whiteListPatterns = new ArrayList<>();
executor = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("RmqMetaReplicator-SourceWatcher-%d").daemon(true).build());
}
@@ -91,7 +94,7 @@ public class RmqMetaReplicator extends SourceConnector {
} catch (IllegalArgumentException e) {
return e.getMessage();
}
-
+ this.prepare();
this.configValid = true;
return "";
}
@@ -99,13 +102,6 @@ public class RmqMetaReplicator extends SourceConnector {
@Override
public void start() {
log.info("starting...");
- try {
- startMQAdminTools();
- } catch (MQClientException e) {
- log.error("Replicator start failed for `startMQAdminTools` exception.", e);
- return;
- }
-
executor.scheduleAtFixedRate(this::refreshConsumerGroups, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
executor.scheduleAtFixedRate(this::syncSubConfig, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
}
@@ -137,13 +133,6 @@ public class RmqMetaReplicator extends SourceConnector {
}
try {
- startMQAdminTools();
- } catch (MQClientException e) {
- log.error("Replicator start failed for `startMQAdminTools` exception.", e);
- throw new IllegalStateException("Replicator start failed for `startMQAdminTools` exception.");
- }
-
- try {
this.syncSubConfig();
this.knownGroups = this.fetchConsumerGroups();
} catch (Exception e) {
@@ -153,14 +142,27 @@ public class RmqMetaReplicator extends SourceConnector {
return Utils.groupPartitions(new ArrayList<>(this.knownGroups), this.replicatorConfig.getTaskParallelism(), replicatorConfig);
}
- private synchronized void startMQAdminTools() throws MQClientException {
+ private void prepare() {
+ this.initWhiteListPatterns();
+ log.info("RocketMQ meta replicator init success for whiteListPatterns.");
+
+ this.startMQAdminTools();
+ log.info("RocketMQ meta replicator start success for mqAdminTools.");
+ }
+
+ private synchronized void startMQAdminTools() {
if (!configValid || adminStarted) {
return;
}
- this.srcMQAdminExt = Utils.startSrcMQAdminTool(this.replicatorConfig);
- this.targetMQAdminExt = Utils.startTargetMQAdminTool(this.replicatorConfig);
- adminStarted = true;
+ try {
+ this.srcMQAdminExt = Utils.startSrcMQAdminTool(this.replicatorConfig);
+ this.targetMQAdminExt = Utils.startTargetMQAdminTool(this.replicatorConfig);
+ this.adminStarted = true;
+ } catch (MQClientException e) {
+ log.error("RocketMQ meta replicator start failed for `startMQAdminTools` exception.", e);
+ throw new IllegalStateException("Replicator start failed for `startMQAdminTools` exception.");
+ }
}
private void refreshConsumerGroups() {
@@ -194,7 +196,7 @@ public class RmqMetaReplicator extends SourceConnector {
String addr = masters.get(0);
SubscriptionGroupWrapper sub = this.srcMQAdminExt.getAllSubscriptionGroup(addr, TimeUnit.SECONDS.toMillis(10));
for (Map.Entry<String, SubscriptionGroupConfig> entry : sub.getSubscriptionGroupTable().entrySet()) {
- if (skipInnerGroup(entry.getKey())) {
+ if (skipInnerGroup(entry.getKey()) || skipNotInWhiteList(entry.getKey())) {
ensureSubConfig(targetBrokers, entry.getValue());
}
}
@@ -211,7 +213,7 @@ public class RmqMetaReplicator extends SourceConnector {
}
private Set<String> fetchConsumerGroups() throws InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
- return listGroups().stream().filter(this::skipInnerGroup).collect(Collectors.toSet());
+ return listGroups().stream().filter(this::skipInnerGroup).filter(this::skipNotInWhiteList).collect(Collectors.toSet());
}
private Set<String> listGroups() throws InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
@@ -232,4 +234,21 @@ public class RmqMetaReplicator extends SourceConnector {
}
return true;
}
+
+ private boolean skipNotInWhiteList(String group) {
+ for (Pattern pattern : this.whiteListPatterns) {
+ Matcher matcher = pattern.matcher(group);
+ if (matcher.matches()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void initWhiteListPatterns() {
+ for (String group : this.replicatorConfig.getWhiteList()) {
+ Pattern pattern = Pattern.compile(group);
+ this.whiteListPatterns.add(pattern);
+ }
+ }
}
[rocketmq-connect] 17/39: Merge pull request #424 from xujianhai666/feat-queuelistener
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit caea266d655b48871e677eb27a98f2a32bc76aed
Merge: 6809200 ebb44c1
Author: Heng Du <du...@apache.org>
AuthorDate: Thu Sep 26 13:49:28 2019 +0800
Merge pull request #424 from xujianhai666/feat-queuelistener
[ISSUE #398]Increase topic change awareness monitoring
.../rocketmq/replicator/RmqSourceReplicator.java | 48 +++++++++++++++++++++-
1 file changed, 47 insertions(+), 1 deletion(-)
[rocketmq-connect] 39/39: Add 'connector/rocketmq-replicator/' from commit '55631bb1fe4d3864c2d45dc7fee9e49764d7b3ac'
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 258e37ef0a7c260f27ac55fab734b836ec4168a5
Merge: ce39378 55631bb
Author: odbozhou <87...@qq.com>
AuthorDate: Wed Mar 2 11:38:45 2022 +0800
Add 'connector/rocketmq-replicator/' from commit '55631bb1fe4d3864c2d45dc7fee9e49764d7b3ac'
git-subtree-dir: connector/rocketmq-replicator
git-subtree-mainline: ce39378b07f04c3deb8ceaed88cd51df7e19381c
git-subtree-split: 55631bb1fe4d3864c2d45dc7fee9e49764d7b3ac
connector/rocketmq-replicator/.gitignore | 13 +
connector/rocketmq-replicator/README.md | 69 +++++
connector/rocketmq-replicator/package.xml | 41 +++
connector/rocketmq-replicator/pom.xml | 154 ++++++++++
.../apache/rocketmq/replicator/MetaSourceTask.java | 156 ++++++++++
.../apache/rocketmq/replicator/RmqConstants.java | 38 +++
.../rocketmq/replicator/RmqMetaReplicator.java | 254 ++++++++++++++++
.../rocketmq/replicator/RmqSourceReplicator.java | 333 +++++++++++++++++++++
.../apache/rocketmq/replicator/RmqSourceTask.java | 273 +++++++++++++++++
.../rocketmq/replicator/common/ConstDefine.java | 29 ++
.../apache/rocketmq/replicator/common/Utils.java | 203 +++++++++++++
.../rocketmq/replicator/config/ConfigDefine.java | 68 +++++
.../rocketmq/replicator/config/ConfigUtil.java | 70 +++++
.../rocketmq/replicator/config/DataType.java | 26 ++
.../replicator/config/RmqConnectorConfig.java | 172 +++++++++++
.../rocketmq/replicator/config/TaskConfig.java | 133 ++++++++
.../rocketmq/replicator/config/TaskConfigEnum.java | 48 +++
.../replicator/config/TaskDivideConfig.java | 123 ++++++++
.../rocketmq/replicator/config/TaskTopicInfo.java | 37 +++
.../rocketmq/replicator/offset/OffsetSync.java | 75 +++++
.../replicator/offset/OffsetSyncStore.java | 90 ++++++
.../rocketmq/replicator/schema/FieldName.java | 32 ++
.../replicator/strategy/DivideStrategyEnum.java | 23 ++
.../strategy/DivideTaskByConsistentHash.java | 89 ++++++
.../replicator/strategy/DivideTaskByQueue.java | 65 ++++
.../replicator/strategy/DivideTaskByTopic.java | 60 ++++
.../replicator/strategy/TaskDivideStrategy.java | 29 ++
.../replicator/RmqSourceReplicatorTest.java | 66 ++++
28 files changed, 2769 insertions(+)
diff --cc connector/rocketmq-replicator/.gitignore
index 0000000,525eaaa..525eaaa
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/.gitignore
+++ b/connector/rocketmq-replicator/.gitignore
diff --cc connector/rocketmq-replicator/README.md
index 0000000,16b01c5..16b01c5
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/README.md
+++ b/connector/rocketmq-replicator/README.md
diff --cc connector/rocketmq-replicator/package.xml
index 0000000,5dc18b9..5dc18b9
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/package.xml
+++ b/connector/rocketmq-replicator/package.xml
diff --cc connector/rocketmq-replicator/pom.xml
index 0000000,c7a08a3..c7a08a3
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/pom.xml
+++ b/connector/rocketmq-replicator/pom.xml
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
index 0000000,67fc89f..67fc89f
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/MetaSourceTask.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
index 0000000,290ab1c..290ab1c
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
index 0000000,bbf3527..bbf3527
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index 0000000,ecbedb6..ecbedb6
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index 0000000,ca7edb4..ca7edb4
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
index 0000000,83583ca..83583ca
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index 0000000,30dc214..30dc214
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
index 0000000,b4de086..b4de086
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/ConfigUtil.java
index 0000000,5da92bc..5da92bc
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/ConfigUtil.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/ConfigUtil.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/DataType.java
index 0000000,60dc330..60dc330
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/DataType.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/DataType.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
index 0000000,2d4fc2a..2d4fc2a
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/RmqConnectorConfig.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
index 0000000,7921585..7921585
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
index 0000000,520c31f..520c31f
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
index 0000000,b8d6fe5..b8d6fe5
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
index 0000000,b791b93..b791b93
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
index 0000000,51c29a7..51c29a7
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSync.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
index 0000000,f8c4b6a..f8c4b6a
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/offset/OffsetSyncStore.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
index 0000000,a8acd08..a8acd08
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideStrategyEnum.java
index 0000000,fb46be3..fb46be3
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideStrategyEnum.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideStrategyEnum.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
index 0000000,708a5b0..708a5b0
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByConsistentHash.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
index 0000000,bbfa580..bbfa580
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
index 0000000,0d13a5e..0d13a5e
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
diff --cc connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
index 0000000,89ed060..89ed060
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
+++ b/connector/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
diff --cc connector/rocketmq-replicator/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
index 0000000,795c386..795c386
mode 000000,100644..100644
--- a/connector/rocketmq-replicator/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
+++ b/connector/rocketmq-replicator/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
[rocketmq-connect] 21/39: [ISSUES #434] Replicator support RocketMQConverter (#463)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit b3377a61e65b81e88824e657f4ddb1f75fb2187f
Author: zhoubo <87...@qq.com>
AuthorDate: Tue Nov 19 23:21:05 2019 +0800
[ISSUES #434] Replicator support RocketMQConverter (#463)
* Replicator support RocketMQConverter
* https://github.com/apache/rocketmq-externals/issues/434
* Replicator converter byte[] data to string
---
.../apache/rocketmq/replicator/RmqSourceTask.java | 24 ++++++++++++----------
1 file changed, 13 insertions(+), 11 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index d965898..b504e85 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -163,17 +163,19 @@ public class RmqSourceTask extends SourceTask {
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis())
.queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
- dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), JSONObject.toJSONString(msgs));
-
- SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap(RmqConstants.getPartition(
- taskTopicConfig.getTopic(),
- taskTopicConfig.getBrokerName(),
- String.valueOf(taskTopicConfig.getQueueId())).getBytes(StandardCharsets.UTF_8)),
- ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8))
- );
- sourceDataEntry.setQueueName(taskTopicConfig.getTargetTopic());
- res.add(sourceDataEntry);
+ for (MessageExt msg : msgs) {
+ dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), new String(msg.getBody()));
+ SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
+ ByteBuffer.wrap(RmqConstants.getPartition(
+ taskTopicConfig.getTopic(),
+ taskTopicConfig.getBrokerName(),
+ String.valueOf(taskTopicConfig.getQueueId())).getBytes(StandardCharsets.UTF_8)),
+ ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8))
+ );
+ sourceDataEntry.setQueueName(taskTopicConfig.getTargetTopic());
+ res.add(sourceDataEntry);
+ }
+
break;
}
default:
[rocketmq-connect] 31/39: docs(replicator): add introduction of replicator (#507)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit b0ca5558acdcf7f3910713b38b756fb69178ce91
Author: ClementIV <fy...@163.com>
AuthorDate: Mon Dec 30 09:52:13 2019 +0800
docs(replicator): add introduction of replicator (#507)
1. rewrite replicator README.md
Close: #505
---
README.md | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 0cb4c9a..171156e 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,28 @@
# rocketmq-replicator
+## rocketmq-replicator简介
+
+
+
+Replicator是RocketMQ Connector的别名，用于RocketMQ集群之间的信息同步，Replicator是运行在RocketMQ Runt上的RocketMQ 集群消息同步Connector，其主要实现了Connector的机制,能够同步两个独立的RocketMQ集群之间的消息。
+
+## 中文文档
+
+[Replicator文档](https://rocketmq-1.gitbook.io/rocketmq-connector/rocketmq-connector/replicator/replicator-jian-jie)
+
+[快速开始](https://rocketmq-1.gitbook.io/rocketmq-connector/rocketmq-connector/replicator/rocketmq-replicator)
+
+---
+
+# replicator使用说明
+
## rocketmq-replicator打包
````
mvn clean install -Prelease-all -DskipTest -U
-打包成功后将rocketmq-replicator-0.1.0-SNAPSHOT-jar-with-dependencies.jar（fatjar）放到runtime配置的pluginPaths目录下
````
+
+打包成功后将`rocketmq-replicator-0.1.0-SNAPSHOT-jar-with-dependencies.jar`（fatjar）放到runtime配置的pluginPaths目录下
+
## rocketmq-replicator启动
同步topic和消息
[rocketmq-connect] 33/39: [Replicator] Fix message duplication problem (#692)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 8e66fd6ec768c6e60e5520e363c46d6c5593370e
Author: Git_Yang <30...@users.noreply.github.com>
AuthorDate: Mon Mar 22 14:38:36 2021 +0800
[Replicator] Fix message duplication problem (#692)
Signed-off-by: zhangyang <Gi...@163.com>
---
src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index 3e8d78b..da7013a 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -160,10 +160,10 @@ public class RmqSourceTask extends SourceTask {
schema.getFields().add(new Field(0,
FieldName.COMMON_MESSAGE.getKey(), FieldType.STRING));
- DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
- dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
for (MessageExt msg : msgs) {
+ DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
+ dataEntryBuilder.timestamp(System.currentTimeMillis())
+ .queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), new String(msg.getBody()));
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
ByteBuffer.wrap(RmqConstants.getPartition(
[rocketmq-connect] 08/39: Update README.md
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit ef2a365ea81798d6478b4d4136158a5e6f4b6a5e
Author: Heng Du <du...@apache.org>
AuthorDate: Wed Aug 14 14:06:54 2019 +0800
Update README.md
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index b2c98d1..332db6e 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@ mvn clean install -Prelease-all -DskipTest -U
## rocketmq-replicator启动
````
http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}
-?config={"connector-class":"org.apache.rocketmq.connector.RmqSourceConnector","source-rocketmq":"xxxx:9876","target-rocketmq":"xxxxxxx:9876","replicator-store-topic":"replicatorTopic","taskDivideStrategy":"0","white-list”:"TopicTest,TopicTest2","task-parallelism":"2","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+?config={"connector-class":"org.apache.rocketmq.replicator.RmqSourceReplicator","source-rocketmq":"xxxx:9876","target-rocketmq":"xxxxxxx:9876","replicator-store-topic":"replicatorTopic","taskDivideStrategy":"0","white-list":"TopicTest,TopicTest2","task-parallelism":"2","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
````
[rocketmq-connect] 16/39: Merge pull request #422 from xujianhai666/fix-emptyTopic
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 6809200a626d97f288ce4e8e0425c368b9fb0170
Merge: 90541a2 d4e677d
Author: Heng Du <du...@apache.org>
AuthorDate: Thu Sep 26 13:47:52 2019 +0800
Merge pull request #422 from xujianhai666/fix-emptyTopic
[ISSUE #386] Fix sourceTaskTopicList is null
src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java | 4 ++++
1 file changed, 4 insertions(+)
[rocketmq-connect] 20/39: feat(replicator): Support subcriptionConfig sync
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit d7b0ce8a8739d5c60390e82cb06375c94e54e565
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Wed Oct 23 16:09:56 2019 +0800
feat(replicator): Support subcriptionConfig sync
- Add syncSubConfig method on RmqMetaReplicator
Closes #438
---
.../rocketmq/replicator/RmqMetaReplicator.java | 91 ++++++++++++++--------
.../apache/rocketmq/replicator/common/Utils.java | 25 ++++++
2 files changed, 85 insertions(+), 31 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
index 38e5af2..3b00de5 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
@@ -20,26 +20,32 @@ import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.source.SourceConnector;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.rocketmq.client.exception.MQBrokerException;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.protocol.body.ClusterInfo;
import org.apache.rocketmq.common.protocol.body.ConsumeStatsList;
-import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.common.protocol.body.SubscriptionGroupWrapper;
+import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig;
import org.apache.rocketmq.remoting.exception.RemotingConnectException;
+import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.remoting.exception.RemotingSendRequestException;
import org.apache.rocketmq.remoting.exception.RemotingTimeoutException;
-import org.apache.rocketmq.replicator.common.ConstDefine;
import org.apache.rocketmq.replicator.common.Utils;
import org.apache.rocketmq.replicator.config.RmqConnectorConfig;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.apache.rocketmq.tools.command.CommandUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,6 +59,7 @@ public class RmqMetaReplicator extends SourceConnector {
private volatile boolean configValid = false;
private Set<String> knownGroups;
private DefaultMQAdminExt srcMQAdminExt;
+ private DefaultMQAdminExt targetMQAdminExt;
private volatile boolean adminStarted;
private ScheduledExecutorService executor;
@@ -92,14 +99,8 @@ public class RmqMetaReplicator extends SourceConnector {
@Override public void start() {
log.info("starting...");
startMQAdminTools();
- executor.scheduleAtFixedRate(() ->
- {
- try {
- refreshConsuemrGroups();
- } catch (Exception e) {
- log.error("refresh consumer groups failed.", e);
- }
- }, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
+ executor.scheduleAtFixedRate(this::refreshConsumerGroups, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
+ executor.scheduleAtFixedRate(this::syncSubConfig, replicatorConfig.getRefreshInterval(), replicatorConfig.getRefreshInterval(), TimeUnit.SECONDS);
}
@Override public void stop() {
@@ -128,6 +129,7 @@ public class RmqMetaReplicator extends SourceConnector {
startMQAdminTools();
try {
+ this.syncSubConfig();
this.knownGroups = this.fetchConsumerGroups();
} catch (Exception e) {
e.printStackTrace();
@@ -140,34 +142,61 @@ public class RmqMetaReplicator extends SourceConnector {
if (!configValid || adminStarted) {
return;
}
- RPCHook rpcHook = null;
- this.srcMQAdminExt = new DefaultMQAdminExt(rpcHook);
- this.srcMQAdminExt.setNamesrvAddr(this.replicatorConfig.getSrcNamesrvs());
- this.srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getSrcNamesrvs()));
try {
- this.srcMQAdminExt.start();
- log.info("RocketMQ srcMQAdminExt started");
+ ImmutablePair<DefaultMQAdminExt, DefaultMQAdminExt> pair = Utils.startMQAdminTools(this.replicatorConfig);
+ this.srcMQAdminExt = pair.getLeft();
+ this.targetMQAdminExt = pair.getRight();
+ log.info("RocketMQ targetMQAdminExt started");
} catch (MQClientException e) {
log.error("Replicator start failed for `srcMQAdminExt` exception.", e);
}
+
adminStarted = true;
}
- private void refreshConsuemrGroups() throws InterruptedException, RemotingConnectException, MQBrokerException, RemotingTimeoutException, MQClientException, RemotingSendRequestException {
- log.debug("refreshConsuemrGroups...");
- Set<String> groups = fetchConsumerGroups();
- Set<String> newGroups = new HashSet<>();
- Set<String> deadGroups = new HashSet<>();
- newGroups.addAll(groups);
- newGroups.removeAll(knownGroups);
- deadGroups.addAll(knownGroups);
- deadGroups.removeAll(groups);
- if (!newGroups.isEmpty() || !deadGroups.isEmpty()) {
- log.info("reconfig consumer groups, new Groups: {} , dead groups: {}, previous groups: {}", newGroups, deadGroups, knownGroups);
- knownGroups = groups;
- context.requestTaskReconfiguration();
+ private void refreshConsumerGroups() {
+ try {
+ log.debug("refreshConsuemrGroups...");
+ Set<String> groups = fetchConsumerGroups();
+ Set<String> newGroups = new HashSet<>(groups);
+ Set<String> deadGroups = new HashSet<>(knownGroups);
+ newGroups.removeAll(knownGroups);
+ deadGroups.removeAll(groups);
+ if (!newGroups.isEmpty() || !deadGroups.isEmpty()) {
+ log.info("reconfig consumer groups, new Groups: {} , dead groups: {}, previous groups: {}", newGroups, deadGroups, knownGroups);
+ knownGroups = groups;
+ context.requestTaskReconfiguration();
+ }
+ } catch (Exception e) {
+ log.error("refresh consumer groups failed.", e);
+ }
+ }
+
+ private void syncSubConfig() {
+ try {
+ Set<String> masterSet =
+ CommandUtil.fetchMasterAddrByClusterName(this.srcMQAdminExt, replicatorConfig.getSrcCluster());
+ List<String> masters = new ArrayList<>(masterSet);
+ Collections.shuffle(masters);
+
+ Set<String> targetBrokers =
+ CommandUtil.fetchMasterAddrByClusterName(this.targetMQAdminExt, replicatorConfig.getSrcCluster());
+
+ String addr = masters.get(0);
+ SubscriptionGroupWrapper sub = this.srcMQAdminExt.getAllSubscriptionGroup(addr, TimeUnit.SECONDS.toMillis(10));
+ for (Map.Entry<String, SubscriptionGroupConfig> entry : sub.getSubscriptionGroupTable().entrySet()) {
+ ensureSubConfig(targetBrokers, entry.getValue());
+ }
+ } catch (Exception e) {
+ log.error("syncSubConfig failed", e);
+ }
+ }
+
+ private void ensureSubConfig(Collection<String> targetBrokers,
+ SubscriptionGroupConfig subConfig) throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
+ for (String addr : targetBrokers) {
+ this.targetMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, subConfig);
}
}
@@ -181,7 +210,7 @@ public class RmqMetaReplicator extends SourceConnector {
String[] addrs = clusterInfo.retrieveAllAddrByCluster(this.replicatorConfig.getSrcCluster());
for (String addr : addrs) {
ConsumeStatsList stats = this.srcMQAdminExt.fetchConsumeStatsInBroker(addr, true, 3 * 1000);
- stats.getConsumeStatsList().stream().map(kv -> kv.keySet()).forEach(groups::addAll);
+ stats.getConsumeStatsList().stream().map(Map::keySet).forEach(groups::addAll);
}
return groups;
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index 4134a2a..60687d7 100644
--- a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
@@ -22,12 +22,15 @@ import io.openmessaging.internal.DefaultKeyValue;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.replicator.config.DataType;
import org.apache.rocketmq.replicator.config.RmqConnectorConfig;
@@ -140,4 +143,26 @@ public class Utils {
return result;
}
+
+ public static ImmutablePair<DefaultMQAdminExt, DefaultMQAdminExt> startMQAdminTools(
+ RmqConnectorConfig replicatorConfig) throws MQClientException {
+ RPCHook rpcHook = null;
+ DefaultMQAdminExt srcMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ srcMQAdminExt.setNamesrvAddr(replicatorConfig.getSrcNamesrvs());
+ srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ srcMQAdminExt.setInstanceName(Utils.createInstanceName(replicatorConfig.getSrcNamesrvs()));
+
+ DefaultMQAdminExt targetMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ targetMQAdminExt.setNamesrvAddr(replicatorConfig.getTargetNamesrvs());
+ targetMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ targetMQAdminExt.setInstanceName(Utils.createInstanceName(replicatorConfig.getTargetNamesrvs()));
+
+ srcMQAdminExt.start();
+ log.info("RocketMQ srcMQAdminExt started");
+
+ targetMQAdminExt.start();
+ log.info("RocketMQ targetMQAdminExt started");
+
+ return ImmutablePair.of(srcMQAdminExt, targetMQAdminExt);
+ }
}
[rocketmq-connect] 34/39: [ISSUE #699] [Replicator] The source task starts to check the position (#700)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit a0b10aad77b0cdfe980b7d6abb1dd3d369689447
Author: Git_Yang <30...@users.noreply.github.com>
AuthorDate: Wed Sep 22 14:09:10 2021 +0800
[ISSUE #699] [Replicator] The source task starts to check the position (#700)
Signed-off-by: zhangyang21 <zh...@xiaomi.com>
---
.../apache/rocketmq/replicator/RmqSourceTask.java | 83 +++++++++++++++++-----
1 file changed, 67 insertions(+), 16 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index da7013a..87ed9a8 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -19,6 +19,7 @@ package org.apache.rocketmq.replicator;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.PositionStorageReader;
import io.openmessaging.connector.api.data.DataEntryBuilder;
import io.openmessaging.connector.api.data.EntryType;
import io.openmessaging.connector.api.data.Field;
@@ -33,6 +34,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.common.message.MessageExt;
@@ -57,6 +59,8 @@ public class RmqSourceTask extends SourceTask {
private final TaskConfig config;
private final DefaultMQPullConsumer consumer;
private volatile boolean started = false;
+ private final long TIMEOUT = 1000 * 60 * 10;
+ private final long WAIT_TIME = 1000 * 2;
private Map<TaskTopicInfo, Long> mqOffsetMap;
@@ -93,27 +97,19 @@ public class RmqSourceTask extends SourceTask {
}
this.consumer.start();
+
+ List<TaskTopicInfo> topicListFilter = new ArrayList<>();
for (TaskTopicInfo tti : topicList) {
Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(tti.getTopic());
for (MessageQueue mq : mqs) {
- if (tti.getQueueId() == mq.getQueueId()) {
- ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
- ByteBuffer.wrap(RmqConstants.getPartition(
- mq.getTopic(),
- mq.getBrokerName(),
- String.valueOf(mq.getQueueId())).getBytes(StandardCharsets.UTF_8)));
-
- if (null != positionInfo && positionInfo.array().length > 0) {
- String positionJson = new String(positionInfo.array(), StandardCharsets.UTF_8);
- JSONObject jsonObject = JSONObject.parseObject(positionJson);
- this.config.setNextPosition(jsonObject.getLong(RmqConstants.NEXT_POSITION));
- } else {
- this.config.setNextPosition(0L);
- }
- mqOffsetMap.put(tti, this.config.getNextPosition());
+ if (tti.getBrokerName().equals(mq.getBrokerName()) && tti.getQueueId() == mq.getQueueId()) {
+ topicListFilter.add(tti);
+ break;
}
}
}
+ PositionStorageReader positionStorageReader = this.context.positionStorageReader();
+ mqOffsetMap.putAll(getPositionMapWithCheck(topicListFilter, positionStorageReader, this.TIMEOUT, TimeUnit.MILLISECONDS));
started = true;
} catch (Exception e) {
log.error("Consumer of task {} start failed.", this.taskId, e);
@@ -163,7 +159,7 @@ public class RmqSourceTask extends SourceTask {
for (MessageExt msg : msgs) {
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
+ .queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), new String(msg.getBody()));
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
ByteBuffer.wrap(RmqConstants.getPartition(
@@ -205,5 +201,60 @@ public class RmqSourceTask extends SourceTask {
private Collection<SourceDataEntry> pollSubConfig() {
return new ArrayList<>();
}
+
+ public Map<TaskTopicInfo, Long> getPositionMapWithCheck(List<TaskTopicInfo> taskList,
+ PositionStorageReader positionStorageReader, long timeout, TimeUnit unit) {
+ unit = unit == null ? TimeUnit.MILLISECONDS : unit;
+
+ Map<TaskTopicInfo, Long> positionMap = getPositionMap(taskList, positionStorageReader);
+
+ long msecs = unit.toMillis(timeout);
+ long startTime = msecs <= 0L ? 0L : System.currentTimeMillis();
+ long waitTime;
+ boolean waitPositionReady;
+ do {
+ try {
+ Thread.sleep(this.WAIT_TIME);
+ } catch (InterruptedException e) {
+ log.error("Thread sleep error.", e);
+ }
+
+ Map<TaskTopicInfo, Long> positionMapCmp = getPositionMap(taskList, positionStorageReader);
+ waitPositionReady = true;
+ for (Map.Entry<TaskTopicInfo, Long> positionEntry : positionMap.entrySet()) {
+ if (positionMapCmp.getOrDefault(positionEntry.getKey(), 0L) != positionEntry.getValue().longValue()) {
+ waitPositionReady = false;
+ positionMap = positionMapCmp;
+ break;
+ }
+ }
+
+ waitTime = msecs - (System.currentTimeMillis() - startTime);
+ } while (!waitPositionReady && waitTime > 0L);
+
+ return positionMap;
+ }
+
+ public Map<TaskTopicInfo, Long> getPositionMap(List<TaskTopicInfo> taskList,
+ PositionStorageReader positionStorageReader) {
+ Map<TaskTopicInfo, Long> positionMap = new HashMap<>();
+ for (TaskTopicInfo tti : taskList) {
+ ByteBuffer positionInfo = positionStorageReader.getPosition(
+ ByteBuffer.wrap(RmqConstants.getPartition(
+ tti.getTopic(),
+ tti.getBrokerName(),
+ String.valueOf(tti.getQueueId())).getBytes(StandardCharsets.UTF_8)));
+
+ if (null != positionInfo && positionInfo.array().length > 0) {
+ String positionJson = new String(positionInfo.array(), StandardCharsets.UTF_8);
+ JSONObject jsonObject = JSONObject.parseObject(positionJson);
+ positionMap.put(tti, jsonObject.getLong(RmqConstants.NEXT_POSITION));
+ } else {
+ positionMap.put(tti, 0L);
+ }
+ }
+
+ return positionMap;
+ }
}
[rocketmq-connect] 37/39: [rocketmq-replicator] Fix topic build route logic (#834)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit d1845c57cb47edb563879586a94a27c7feab7096
Author: Git_Yang <30...@users.noreply.github.com>
AuthorDate: Wed Oct 27 14:18:58 2021 +0800
[rocketmq-replicator] Fix topic build route logic (#834)
Signed-off-by: zhangyang21 <zh...@xiaomi.com>
---
.../rocketmq/replicator/RmqSourceReplicator.java | 65 ++++++++++++----------
1 file changed, 36 insertions(+), 29 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index 50f4a17..ecbedb6 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -228,37 +228,44 @@ public class RmqSourceReplicator extends SourceConnector {
TopicList topics = srcMQAdminExt.fetchAllTopicList();
for (String topic : topics.getTopicList()) {
- if ((syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) ||
- (syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) ||
- !topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
-
- for (Pattern pattern : patterns) {
- Matcher matcher = pattern.matcher(topic);
- if (matcher.matches()) {
- String targetTopic = generateTargetTopic(topic);
- if (!targetTopicSet.contains(targetTopic)) {
- ensureTargetTopic(topic, targetTopic);
- }
+ if (topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
+ continue;
+ }
- // different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
- // QueueData use brokerName as unique info on cluster of rocketmq. so when we want to get QueueData of
- // expected cluster, we should get brokerNames of expected cluster, and then filter queueDatas.
- List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, topic, srcCluster);
- Set<String> brokerNameSet = new HashSet<String>();
- for (BrokerData b : brokerList) {
- brokerNameSet.add(b.getBrokerName());
- }
+ if (!syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
+ continue;
+ }
- TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
- if (!topicRouteMap.containsKey(topic)) {
- topicRouteMap.put(topic, new HashSet<>(16));
- }
- for (QueueData qd : topicRouteData.getQueueDatas()) {
- if (brokerNameSet.contains(qd.getBrokerName())) {
- for (int i = 0; i < qd.getReadQueueNums(); i++) {
- TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), i, targetTopic);
- topicRouteMap.get(topic).add(taskTopicInfo);
- }
+ if (!syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) {
+ continue;
+ }
+
+ for (Pattern pattern : patterns) {
+ Matcher matcher = pattern.matcher(topic);
+ if (matcher.matches()) {
+ String targetTopic = generateTargetTopic(topic);
+ if (!targetTopicSet.contains(targetTopic)) {
+ ensureTargetTopic(topic, targetTopic);
+ }
+
+ // different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
+ // QueueData use brokerName as unique info on cluster of rocketmq. so when we want to get QueueData of
+ // expected cluster, we should get brokerNames of expected cluster, and then filter queueDatas.
+ List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, topic, srcCluster);
+ Set<String> brokerNameSet = new HashSet<String>();
+ for (BrokerData b : brokerList) {
+ brokerNameSet.add(b.getBrokerName());
+ }
+
+ TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
+ if (!topicRouteMap.containsKey(topic)) {
+ topicRouteMap.put(topic, new HashSet<>(16));
+ }
+ for (QueueData qd : topicRouteData.getQueueDatas()) {
+ if (brokerNameSet.contains(qd.getBrokerName())) {
+ for (int i = 0; i < qd.getReadQueueNums(); i++) {
+ TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), i, targetTopic);
+ topicRouteMap.get(topic).add(taskTopicInfo);
}
}
}
[rocketmq-connect] 15/39: Increase topic change awareness monitoring. resolve #398
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit ebb44c150bc43fe187c28b44e0b1915c2e901dfb
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Sun Sep 22 10:26:09 2019 +0800
Increase topic change awareness monitoring. resolve #398
---
.../rocketmq/replicator/RmqSourceReplicator.java | 48 +++++++++++++++++++++-
1 file changed, 47 insertions(+), 1 deletion(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index e124f15..429102e 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -25,14 +25,18 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.commons.lang3.text.StrSubstitutor;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.TopicConfig;
-import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.body.TopicList;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.QueueData;
@@ -78,9 +82,12 @@ public class RmqSourceReplicator extends SourceConnector {
private volatile boolean adminStarted;
+ private ScheduledExecutorService executor;
+
public RmqSourceReplicator() {
topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
whiteList = new HashSet<String>();
+ executor = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("RmqSourceReplicator-SourceWatcher-%d").daemon(true).build());
}
private synchronized void startMQAdminTools() {
@@ -151,6 +158,44 @@ public class RmqSourceReplicator extends SourceConnector {
@Override
public void start() {
startMQAdminTools();
+ startListner();
+ }
+
+ public void startListner() {
+ executor.scheduleAtFixedRate(new Runnable() {
+ @Override public void run() {
+ Map<String, List<TaskTopicInfo>> origin = topicRouteMap;
+ topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
+
+ buildRoute();
+
+ if (!compare(origin, topicRouteMap)) {
+ context.requestTaskReconfiguration();
+ }
+ }
+ }, 30, 30, TimeUnit.SECONDS);
+ }
+
+ public boolean compare(Map<String, List<TaskTopicInfo>> origin, Map<String, List<TaskTopicInfo>> updated) {
+ if (origin.size() != updated.size()) {
+ return false;
+ }
+ for (Map.Entry<String, List<TaskTopicInfo>> entry : origin.entrySet()) {
+ if (!updated.containsKey(entry.getKey())) {
+ return false;
+ }
+ List<TaskTopicInfo> originTasks = entry.getValue();
+ List<TaskTopicInfo> updateTasks = updated.get(entry.getKey());
+ if (originTasks.size() != updateTasks.size()) {
+ return false;
+ }
+
+ if (!originTasks.containsAll(updateTasks)) {
+ return false;
+ }
+ }
+
+ return true;
}
public void stop() {
@@ -169,6 +214,7 @@ public class RmqSourceReplicator extends SourceConnector {
return RmqSourceTask.class;
}
+ @Override
public List<KeyValue> taskConfigs() {
if (!configValid) {
return new ArrayList<KeyValue>();
[rocketmq-connect] 26/39: [ISSUE #468] Replicator is packaged as fatjar (#472)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 7201d00f7d30b35f8e00ec179b1dce46ffea85fa
Author: zhoubo <87...@qq.com>
AuthorDate: Thu Dec 5 09:56:16 2019 +0800
[ISSUE #468] Replicator is packaged as fatjar (#472)
* Replicator is packaged as fatjar
https://github.com/apache/rocketmq-externals/issues/468
* Upgrade openmessaging-connector version
---
pom.xml | 22 +++++++++++++++++++++-
1 file changed, 21 insertions(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index fc35065..c7a08a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,6 +34,25 @@
<target>8</target>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>3.2.0</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>assemble-all</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -45,7 +64,8 @@
<dependency>
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-connector</artifactId>
- <version>0.1.1-beta-SNAPSHOT</version>
+ <version>0.1.1</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
[rocketmq-connect] 32/39: Update README.md
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 2e8ea2ed27277fd9760e3ad224682c2f95bc74cd
Author: Heng Du <du...@apache.org>
AuthorDate: Sat Mar 21 19:05:44 2020 +0800
Update README.md
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 171156e..16b01c5 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
-Replicator是RocketMQ Connector的别名，用于RocketMQ集群之间的信息同步，Replicator是运行在RocketMQ Runt上的RocketMQ 集群消息同步Connector，其主要实现了Connector的机制,能够同步两个独立的RocketMQ集群之间的消息。
+Replicator用于RocketMQ集群之间的信息同步，作为一个connector运行在RocketMQ Runtime上之上, 能够同步两个独立的RocketMQ集群之间的消息。
## 中文文档
[rocketmq-connect] 06/39: 【ISSUE #278】Define and Implement the RmqConnector and RmqSourceTask. (#381)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit c1a51cc8e2223ea43534a9010a86b6930d391675
Author: chuenfaiy <ch...@163.com>
AuthorDate: Tue Aug 13 11:40:22 2019 +0800
【ISSUE #278】Define and Implement the RmqConnector and RmqSourceTask. (#381)
* init commit
* [Fix]fix some bugs in connectors and fastjson.
* [Update]update the version of replicator
* [Update]remove the repetitive properties
* [Update]Add the license at the head of TaskTopicInfo class and TaskConfigEnum class
* [Update]update the project name from to
* [Update]update the language of README.md
* [Update]change the title of README.md
* [Update]change the package name from to
* [Update]update the starting position
* [Update]update the starting position
* [Fix]fix the bug of instanceName setting
---
README.md | 18 ++--
pom.xml | 4 +-
.../{connector => replicator}/RmqConstants.java | 4 +-
.../RmqSourceReplicator.java} | 119 ++++++++++-----------
.../{connector => replicator}/RmqSourceTask.java | 14 +--
.../common/ConstDefine.java | 2 +-
.../{connector => replicator}/common/Utils.java | 2 +-
.../config/ConfigDefine.java | 2 +-
.../config/ConfigUtil.java | 2 +-
.../{connector => replicator}/config/DataType.java | 2 +-
.../config/TaskConfig.java | 2 +-
.../config/TaskConfigEnum.java | 2 +-
.../config/TaskDivideConfig.java | 2 +-
.../config/TaskTopicInfo.java | 2 +-
.../schema/FieldName.java | 2 +-
.../strategy/DivideStrategyEnum.java | 2 +-
.../strategy/DivideTaskByQueue.java | 8 +-
.../strategy/DivideTaskByTopic.java | 5 +-
.../strategy/TaskDivideStrategy.java | 5 +-
19 files changed, 95 insertions(+), 104 deletions(-)
diff --git a/README.md b/README.md
index 0b8bef7..77bcb94 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,13 @@
# rocketmq-replicator
-# 启动参数选择
+# boot-parameter
-参数 | 类型 |是否必须 |说明|示例值
+parameter | type | must | description | sample value
---|---|---|---|---|
-source-rocketmq | 字符串 | 是 | 源rocketmq集群namesrv地址 | 192.168.1.2:9876 |
-target-rocketmq | 字符串 | 是 | 源rocketmq集群namesrv地址 | 192.168.1.2:9876 |
-replicator-store-topic | 字符串 | 是 | replicator存储topic，需要在runtime的mq集群提前创建 | replicator-store-topic |
-task-divide-strategy | 整型 | 否 | 任务切割策略，可以按照主题和队列来切割，目前只支持主题切割且主题对应值为0 | 0 |
-white-list | 字符串 | 是 | 复制主题白名单，多个topic之间使用逗号分隔 | topic-1,topic-2 |
-task-parallelism | 整型 | 否 | 任务并行度，默认值为1，当topic数大于task数时，一个task将负责多个topic | 2 |
-source-record-converter | 字符串 | 是 | 源数据解析器，目前使用的是Json解析器 | io.openmessaging.connect.runtime.converter.JsonConverter |
\ No newline at end of file
+source-rocketmq | String | Yes | namesrv address of source rocketmq cluster | 192.168.1.2:9876 |
+target-rocketmq | String | Yes | namesrv address of target rocketmq cluster | 192.168.1.2:9876 |
+replicator-store-topic | String | Yes | topic name to store all source messages | replicator-store-topic |
+task-divide-strategy | Integer | No | task dividing strategy, default value is 0 for dividing by topic | 0 |
+white-list | String | Yes | topic white list and multiple fields are separated by commas | topic-1,topic-2 |
+task-parallelism | String | No | task parallelism，default value is 1，one task will be responsible for multiple topics for the value greater than 1 | 2 |
+source-record-converter | String | Yes | source data parser | io.openmessaging.connect.runtime.converter.JsonConverter |
diff --git a/pom.xml b/pom.xml
index 37bf78f..d283ca9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,8 +22,8 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.rocketmq</groupId>
- <artifactId>rocketmq-connector</artifactId>
- <version>0.0.1-SNAPSHOT</version>
+ <artifactId>rocketmq-replicator</artifactId>
+ <version>0.1.0-SNAPSHOT</version>
<properties>
<rocketmq.version>4.4.0</rocketmq.version>
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqConstants.java b/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
similarity index 90%
rename from src/main/java/org/apache/rocketmq/connector/RmqConstants.java
rename to src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
index 1278424..4994abe 100644
--- a/src/main/java/org/apache/rocketmq/connector/RmqConstants.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqConstants.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector;
+package org.apache.rocketmq.replicator;
public class RmqConstants {
@@ -26,6 +26,8 @@ public class RmqConstants {
public static final String NEXT_POSITION = "nextPosition";
+ public static final String SOURCE_INSTANCE_NAME = "REPLICATOR_SOURCE_CONSUMER";
+
public static String getPartition(String topic, String broker, String queueId) {
return new StringBuilder().append(broker).append(topic).append(queueId).toString();
}
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
similarity index 54%
rename from src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
rename to src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index 33eb958..c1f350f 100644
--- a/src/main/java/org/apache/rocketmq/connector/RmqSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector;
+package org.apache.rocketmq.replicator;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
@@ -24,15 +24,15 @@ import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageQueue;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
-import org.apache.rocketmq.connector.common.ConstDefine;
-import org.apache.rocketmq.connector.common.Utils;
-import org.apache.rocketmq.connector.config.ConfigDefine;
-import org.apache.rocketmq.connector.config.DataType;
-import org.apache.rocketmq.connector.config.TaskDivideConfig;
-import org.apache.rocketmq.connector.strategy.DivideStrategyEnum;
-import org.apache.rocketmq.connector.strategy.DivideTaskByQueue;
-import org.apache.rocketmq.connector.strategy.DivideTaskByTopic;
-import org.apache.rocketmq.connector.strategy.TaskDivideStrategy;
+import org.apache.rocketmq.replicator.common.ConstDefine;
+import org.apache.rocketmq.replicator.common.Utils;
+import org.apache.rocketmq.replicator.config.ConfigDefine;
+import org.apache.rocketmq.replicator.config.DataType;
+import org.apache.rocketmq.replicator.config.TaskDivideConfig;
+import org.apache.rocketmq.replicator.strategy.DivideStrategyEnum;
+import org.apache.rocketmq.replicator.strategy.DivideTaskByQueue;
+import org.apache.rocketmq.replicator.strategy.DivideTaskByTopic;
+import org.apache.rocketmq.replicator.strategy.TaskDivideStrategy;
import org.apache.rocketmq.remoting.RPCHook;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.slf4j.Logger;
@@ -40,15 +40,15 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-public class RmqSourceConnector extends SourceConnector {
+public class RmqSourceReplicator extends SourceConnector {
- private static final Logger log = LoggerFactory.getLogger(RmqSourceConnector.class);
+ private static final Logger log = LoggerFactory.getLogger(RmqSourceReplicator.class);
private boolean syncDLQ = false;
private boolean syncRETRY = false;
- private KeyValue config;
+ private KeyValue replicatorConfig;
private Map<String, List<MessageQueue>> topicRouteMap;
@@ -56,15 +56,11 @@ public class RmqSourceConnector extends SourceConnector {
private Set<String> whiteList;
- private volatile boolean started = false;
-
private volatile boolean configValid = false;
- private DefaultMQAdminExt defaultMQAdminExt;
-
private int taskParallelism = 1;
- public RmqSourceConnector() {
+ public RmqSourceReplicator() {
topicRouteMap = new HashMap<String, List<MessageQueue>>();
whiteList = new HashSet<String>();
@@ -80,7 +76,7 @@ public class RmqSourceConnector extends SourceConnector {
}
// check the whitelist, whitelist is required.
- String whileListStr = this.config.getString(ConfigDefine.CONN_WHITE_LIST);
+ String whileListStr = config.getString(ConfigDefine.CONN_WHITE_LIST);
String[] wl = whileListStr.trim().split(",");
if (wl.length <= 0) return "White list must be not empty.";
else {
@@ -89,46 +85,26 @@ public class RmqSourceConnector extends SourceConnector {
}
}
- if (this.config.containsKey(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) &&
- this.config.getInt(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) == DivideStrategyEnum.BY_QUEUE.ordinal()) {
+ if (config.containsKey(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) &&
+ config.getInt(ConfigDefine.CONN_TASK_DIVIDE_STRATEGY) == DivideStrategyEnum.BY_QUEUE.ordinal()) {
this.taskDivideStrategy = new DivideTaskByQueue();
} else {
this.taskDivideStrategy = new DivideTaskByTopic();
}
if (config.containsKey(ConfigDefine.CONN_TASK_PARALLELISM)) {
- this.taskParallelism = this.config.getInt(ConfigDefine.CONN_TASK_PARALLELISM);
+ this.taskParallelism = config.getInt(ConfigDefine.CONN_TASK_PARALLELISM);
}
- this.config = config;
+ this.replicatorConfig = config;
this.configValid = true;
return "";
}
public void start() {
-
- if (configValid) {
- RPCHook rpcHook = null;
- this.defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
- this.defaultMQAdminExt.setNamesrvAddr(this.config.getString(ConfigDefine.CONN_SOURCE_RMQ));
- this.defaultMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
- this.defaultMQAdminExt.setInstanceName(Utils.createInstanceName(this.config.getString(ConfigDefine.CONN_SOURCE_RMQ)));
- try {
- defaultMQAdminExt.start();
- } catch (MQClientException e) {
- log.error("Replicator start failed for `defaultMQAdminExt` exception.", e);
- }
- started = true;
- }
}
public void stop() {
- if (started) {
- if (defaultMQAdminExt != null) {
- defaultMQAdminExt.shutdown();
- }
- started = false;
- }
}
public void pause() {
@@ -146,33 +122,52 @@ public class RmqSourceConnector extends SourceConnector {
public List<KeyValue> taskConfigs() {
- if (started && configValid) {
+ if (configValid) {
+
+ boolean adminStarted = false;
+ RPCHook rpcHook = null;
+ DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ defaultMQAdminExt.setNamesrvAddr(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ));
+ defaultMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.REPLICATOR_ADMIN_PREFIX));
+ defaultMQAdminExt.setInstanceName(Utils.createInstanceName(this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ)));
+
try {
- for (String topic : this.whiteList) {
- if ((syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) ||
- (syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) ||
- !topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
-
- TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
- if (!topicRouteMap.containsKey(topic)) {
- topicRouteMap.put(topic, new ArrayList<MessageQueue>());
- }
- for (QueueData qd : topicRouteData.getQueueDatas()) {
- for (int i = 0; i < qd.getReadQueueNums(); i++) {
- MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
- topicRouteMap.get(topic).add(mq);
+ defaultMQAdminExt.start();
+ adminStarted = true;
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `defaultMQAdminExt` exception.", e);
+ }
+
+ if (adminStarted) {
+ try {
+ for (String topic : this.whiteList) {
+ if ((syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) ||
+ (syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) ||
+ !topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
+
+ TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+ if (!topicRouteMap.containsKey(topic)) {
+ topicRouteMap.put(topic, new ArrayList<MessageQueue>());
+ }
+ for (QueueData qd : topicRouteData.getQueueDatas()) {
+ for (int i = 0; i < qd.getReadQueueNums(); i++) {
+ MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
+ topicRouteMap.get(topic).add(mq);
+ }
}
}
}
+ } catch (Exception e) {
+ log.error("Fetch topic list error.", e);
+ } finally {
+ defaultMQAdminExt.shutdown();
}
- } catch (Exception e) {
- log.error("Fetch topic list error.", e);
}
TaskDivideConfig tdc = new TaskDivideConfig(
- this.config.getString(ConfigDefine.CONN_SOURCE_RMQ),
- this.config.getString(ConfigDefine.CONN_STORE_TOPIC),
- this.config.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER),
+ this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ),
+ this.replicatorConfig.getString(ConfigDefine.CONN_STORE_TOPIC),
+ this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER),
DataType.COMMON_MESSAGE.ordinal(),
this.taskParallelism
);
diff --git a/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
similarity index 95%
rename from src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
rename to src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index 7b3b011..50a88bb 100644
--- a/src/main/java/org/apache/rocketmq/connector/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector;
+package org.apache.rocketmq.replicator;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
@@ -25,12 +25,12 @@ import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.consumer.PullResult;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.connector.common.Utils;
-import org.apache.rocketmq.connector.config.ConfigUtil;
-import org.apache.rocketmq.connector.config.DataType;
-import org.apache.rocketmq.connector.config.TaskConfig;
-import org.apache.rocketmq.connector.config.TaskTopicInfo;
-import org.apache.rocketmq.connector.schema.FieldName;
+import org.apache.rocketmq.replicator.common.Utils;
+import org.apache.rocketmq.replicator.config.ConfigUtil;
+import org.apache.rocketmq.replicator.config.DataType;
+import org.apache.rocketmq.replicator.config.TaskConfig;
+import org.apache.rocketmq.replicator.config.TaskTopicInfo;
+import org.apache.rocketmq.replicator.schema.FieldName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/main/java/org/apache/rocketmq/connector/common/ConstDefine.java b/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
similarity index 95%
rename from src/main/java/org/apache/rocketmq/connector/common/ConstDefine.java
rename to src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
index 6e1eeac..08d985b 100644
--- a/src/main/java/org/apache/rocketmq/connector/common/ConstDefine.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.common;
+package org.apache.rocketmq.replicator.common;
public class ConstDefine {
diff --git a/src/main/java/org/apache/rocketmq/connector/common/Utils.java b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
similarity index 97%
rename from src/main/java/org/apache/rocketmq/connector/common/Utils.java
rename to src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index 30fe44c..0e4766d 100644
--- a/src/main/java/org/apache/rocketmq/connector/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.common;
+package org.apache.rocketmq.replicator.common;
import java.util.ArrayList;
import java.util.Collections;
diff --git a/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
similarity index 97%
rename from src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java
rename to src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
index 13f8960..3934c2f 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/ConfigDefine.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/ConfigDefine.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.config;
+package org.apache.rocketmq.replicator.config;
import java.util.HashSet;
import java.util.Set;
diff --git a/src/main/java/org/apache/rocketmq/connector/config/ConfigUtil.java b/src/main/java/org/apache/rocketmq/replicator/config/ConfigUtil.java
similarity index 98%
rename from src/main/java/org/apache/rocketmq/connector/config/ConfigUtil.java
rename to src/main/java/org/apache/rocketmq/replicator/config/ConfigUtil.java
index d2dd7da..5da92bc 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/ConfigUtil.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/ConfigUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.config;
+package org.apache.rocketmq.replicator.config;
import io.openmessaging.KeyValue;
diff --git a/src/main/java/org/apache/rocketmq/connector/config/DataType.java b/src/main/java/org/apache/rocketmq/replicator/config/DataType.java
similarity index 95%
rename from src/main/java/org/apache/rocketmq/connector/config/DataType.java
rename to src/main/java/org/apache/rocketmq/replicator/config/DataType.java
index 3e77a3a..75f772e 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/DataType.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/DataType.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.config;
+package org.apache.rocketmq.replicator.config;
public enum DataType {
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
similarity index 97%
rename from src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
rename to src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
index d480b90..e85280f 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/TaskConfig.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfig.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.config;
+package org.apache.rocketmq.replicator.config;
public class TaskConfig {
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
similarity index 96%
rename from src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java
rename to src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
index fca4dcc..1516f7a 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/TaskConfigEnum.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskConfigEnum.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.config;
+package org.apache.rocketmq.replicator.config;
public enum TaskConfigEnum {
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskDivideConfig.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
similarity index 98%
rename from src/main/java/org/apache/rocketmq/connector/config/TaskDivideConfig.java
rename to src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
index 7f904b3..e6a8144 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/TaskDivideConfig.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskDivideConfig.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.config;
+package org.apache.rocketmq.replicator.config;
public class TaskDivideConfig {
diff --git a/src/main/java/org/apache/rocketmq/connector/config/TaskTopicInfo.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
similarity index 97%
rename from src/main/java/org/apache/rocketmq/connector/config/TaskTopicInfo.java
rename to src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
index e1f47d5..f078028 100644
--- a/src/main/java/org/apache/rocketmq/connector/config/TaskTopicInfo.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.config;
+package org.apache.rocketmq.replicator.config;
public class TaskTopicInfo {
diff --git a/src/main/java/org/apache/rocketmq/connector/schema/FieldName.java b/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
similarity index 95%
rename from src/main/java/org/apache/rocketmq/connector/schema/FieldName.java
rename to src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
index 9f4dc47..913ffca 100644
--- a/src/main/java/org/apache/rocketmq/connector/schema/FieldName.java
+++ b/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.schema;
+package org.apache.rocketmq.replicator.schema;
public enum FieldName {
COMMON_MESSAGE("MessageExt");
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideStrategyEnum.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideStrategyEnum.java
similarity index 94%
rename from src/main/java/org/apache/rocketmq/connector/strategy/DivideStrategyEnum.java
rename to src/main/java/org/apache/rocketmq/replicator/strategy/DivideStrategyEnum.java
index 9dc060f..fb46be3 100644
--- a/src/main/java/org/apache/rocketmq/connector/strategy/DivideStrategyEnum.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideStrategyEnum.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.strategy;
+package org.apache.rocketmq.replicator.strategy;
public enum DivideStrategyEnum {
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
similarity index 83%
rename from src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java
rename to src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
index caa69f3..77ed871 100644
--- a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByQueue.java
@@ -14,15 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.strategy;
+package org.apache.rocketmq.replicator.strategy;
import io.openmessaging.KeyValue;
-import io.openmessaging.internal.DefaultKeyValue;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.connector.config.ConfigDefine;
-import org.apache.rocketmq.connector.config.DataType;
-import org.apache.rocketmq.connector.config.TaskDivideConfig;
-
+import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
similarity index 96%
rename from src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
rename to src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
index 9b019f1..e667d57 100644
--- a/src/main/java/org/apache/rocketmq/connector/strategy/DivideTaskByTopic.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/DivideTaskByTopic.java
@@ -14,14 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.strategy;
+package org.apache.rocketmq.replicator.strategy;
import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.internal.DefaultKeyValue;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.connector.config.*;
-
+import org.apache.rocketmq.replicator.config.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
diff --git a/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java b/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
similarity index 90%
rename from src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java
rename to src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
index e80d092..0e0ac99 100644
--- a/src/main/java/org/apache/rocketmq/connector/strategy/TaskDivideStrategy.java
+++ b/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java
@@ -14,12 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.rocketmq.connector.strategy;
+package org.apache.rocketmq.replicator.strategy;
import io.openmessaging.KeyValue;
import org.apache.rocketmq.common.message.MessageQueue;
-import org.apache.rocketmq.connector.config.TaskDivideConfig;
-
+import org.apache.rocketmq.replicator.config.TaskDivideConfig;
import java.util.List;
import java.util.Map;
[rocketmq-connect] 09/39: Support wildcard subscription topic. resolve #395
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit d80142a96c9b1f43f9bbbdbef85df6e7ad3501e2
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Sun Sep 8 01:13:11 2019 +0800
Support wildcard subscription topic. resolve #395
---
pom.xml | 14 +++-
.../rocketmq/replicator/RmqSourceReplicator.java | 59 ++++++++++----
.../apache/rocketmq/replicator/RmqSourceTask.java | 2 +-
.../replicator/RmqSourceReplicatorTest.java | 92 ++++++++++++++++++++++
4 files changed, 149 insertions(+), 18 deletions(-)
diff --git a/pom.xml b/pom.xml
index 963222e..b528c80 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,6 +42,18 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>2.6.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <version>2.6.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.5</version>
@@ -106,4 +118,4 @@
</profile>
</profiles>
-</project>
\ No newline at end of file
+</project>
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index 12473ab..b49e06d 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -25,9 +25,12 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.MixAll;
import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.body.TopicList;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
import org.apache.rocketmq.remoting.RPCHook;
@@ -156,20 +159,45 @@ public class RmqSourceReplicator extends SourceConnector {
startMQAdminTools();
+ buildRoute();
+
+ TaskDivideConfig tdc = new TaskDivideConfig(
+ this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ),
+ this.replicatorConfig.getString(ConfigDefine.CONN_STORE_TOPIC),
+ this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER),
+ DataType.COMMON_MESSAGE.ordinal(),
+ this.taskParallelism
+ );
+ return this.taskDivideStrategy.divide(this.topicRouteMap, tdc);
+ }
+
+ public void buildRoute() {
try {
+ List<Pattern> patterns = new ArrayList<Pattern>();
for (String topic : this.whiteList) {
+ Pattern pattern = Pattern.compile(topic);
+ patterns.add(pattern);
+ }
+
+ TopicList topics = defaultMQAdminExt.fetchAllTopicList();
+ for (String topic : topics.getTopicList()) {
if ((syncRETRY && topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) ||
(syncDLQ && topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) ||
!topic.equals(ConfigDefine.CONN_STORE_TOPIC)) {
- TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
- if (!topicRouteMap.containsKey(topic)) {
- topicRouteMap.put(topic, new ArrayList<MessageQueue>());
- }
- for (QueueData qd : topicRouteData.getQueueDatas()) {
- for (int i = 0; i < qd.getReadQueueNums(); i++) {
- MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
- topicRouteMap.get(topic).add(mq);
+ for (Pattern pattern : patterns) {
+ Matcher matcher = pattern.matcher(topic);
+ if (matcher.matches()) {
+ TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+ if (!topicRouteMap.containsKey(topic)) {
+ topicRouteMap.put(topic, new ArrayList<MessageQueue>());
+ }
+ for (QueueData qd : topicRouteData.getQueueDatas()) {
+ for (int i = 0; i < qd.getReadQueueNums(); i++) {
+ MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i);
+ topicRouteMap.get(topic).add(mq);
+ }
+ }
}
}
}
@@ -179,15 +207,14 @@ public class RmqSourceReplicator extends SourceConnector {
} finally {
defaultMQAdminExt.shutdown();
}
+ }
- TaskDivideConfig tdc = new TaskDivideConfig(
- this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RMQ),
- this.replicatorConfig.getString(ConfigDefine.CONN_STORE_TOPIC),
- this.replicatorConfig.getString(ConfigDefine.CONN_SOURCE_RECORD_CONVERTER),
- DataType.COMMON_MESSAGE.ordinal(),
- this.taskParallelism
- );
- return this.taskDivideStrategy.divide(this.topicRouteMap, tdc);
+ public void setWhiteList(Set<String> whiteList) {
+ this.whiteList = whiteList;
+ }
+
+ public Map<String, List<MessageQueue>> getTopicRouteMap() {
+ return this.topicRouteMap;
}
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index 8c8e434..9909d61 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -126,7 +126,7 @@ public class RmqSourceTask extends SourceTask {
}
public void stop() {
-
+
if (started) {
if (this.consumer != null) {
this.consumer.shutdown();
diff --git a/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java b/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
new file mode 100644
index 0000000..6ac0049
--- /dev/null
+++ b/src/test/java/org/apache/rocketmq/replicator/RmqSourceReplicatorTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.replicator;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.message.MessageQueue;
+import org.apache.rocketmq.common.protocol.body.TopicList;
+import org.apache.rocketmq.common.protocol.route.QueueData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.FieldSetter;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class RmqSourceReplicatorTest {
+
+ @Mock
+ private DefaultMQAdminExt defaultMQAdminExt;
+
+
+ @Test
+ public void buildWildcardRoute() throws RemotingException, MQClientException, InterruptedException, NoSuchFieldException {
+
+ RmqSourceReplicator rmqSourceReplicator = Mockito.spy(RmqSourceReplicator.class);
+
+ TopicList topicList = new TopicList();
+ Set<String> topics = new HashSet<String>();
+ topics.add("topic1");
+ topics.add("topic2");
+ topics.add("sub-topic1-test");
+ topics.add("sub-topic2-test");
+ topics.add("sub-topic2-xxx");
+ topics.add("sub-0");
+ topics.add("test-0");
+ topics.add("0-test");
+ topicList.setTopicList(topics);
+ when(defaultMQAdminExt.fetchAllTopicList()).thenReturn(topicList);
+
+ TopicRouteData topicRouteData = new TopicRouteData();
+ topicRouteData.setQueueDatas(Collections.<QueueData>emptyList());
+ when(defaultMQAdminExt.examineTopicRouteInfo(any(String.class))).thenReturn(topicRouteData);
+
+
+ Field field = RmqSourceReplicator.class.getDeclaredField("defaultMQAdminExt");
+ FieldSetter.setField(rmqSourceReplicator, field, defaultMQAdminExt);
+
+ Set<String> whiteList = new HashSet<String>();
+ whiteList.add("topic1");
+ whiteList.add("\\w+-test");
+ rmqSourceReplicator.setWhiteList(whiteList);
+ rmqSourceReplicator.buildRoute();
+ Map<String, List<MessageQueue>> queues = rmqSourceReplicator.getTopicRouteMap();
+ Set<String> expected = new HashSet<String>();
+ expected.add("topic1");
+ expected.add("0-test");
+ assertThat(queues.size()).isEqualTo(expected.size());
+ for (String topic : expected) {
+ assertThat(queues.containsKey(topic)).isTrue();
+ }
+ }
+}
[rocketmq-connect] 12/39: extend messageQueue for TaskTopicInfo
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit ad38ec76cc737aef32fc5c5e8533af6f19fc72b4
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Tue Sep 17 14:24:31 2019 +0800
extend messageQueue for TaskTopicInfo
---
.../rocketmq/replicator/RmqSourceReplicator.java | 2 +-
.../apache/rocketmq/replicator/RmqSourceTask.java | 59 +++++++---------------
.../rocketmq/replicator/config/TaskTopicInfo.java | 40 ++-------------
3 files changed, 23 insertions(+), 78 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
index f24cf63..e124f15 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceReplicator.java
@@ -228,7 +228,7 @@ public class RmqSourceReplicator extends SourceConnector {
for (QueueData qd : topicRouteData.getQueueDatas()) {
if (brokerNameSet.contains(qd.getBrokerName())) {
for (int i = 0; i < qd.getReadQueueNums(); i++) {
- TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), String.valueOf(i), targetTopic);
+ TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), i, targetTopic);
topicRouteMap.get(topic).add(taskTopicInfo);
}
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index c4d53fa..b16e585 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -47,6 +47,7 @@ public class RmqSourceTask extends SourceTask {
private volatile boolean started = false;
private Map<TaskTopicInfo, Long> mqOffsetMap;
+
public RmqSourceTask() {
this.config = new TaskConfig();
this.consumer = new DefaultMQPullConsumer();
@@ -76,36 +77,15 @@ public class RmqSourceTask extends SourceTask {
try {
this.consumer.start();
- for (TaskTopicInfo tti: topicList) {
- Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(tti.getSourceTopic());
- if (!tti.getQueueId().equals("")) {
- // divide task by queue
- for (MessageQueue mq: mqs) {
- if (Integer.valueOf(tti.getQueueId()) == mq.getQueueId()) {
- ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
- ByteBuffer.wrap(RmqConstants.getPartition(
- mq.getTopic(),
- mq.getBrokerName(),
- String.valueOf(mq.getQueueId())).getBytes("UTF-8")));
-
- if (null != positionInfo && positionInfo.array().length > 0) {
- String positionJson = new String(positionInfo.array(), "UTF-8");
- JSONObject jsonObject = JSONObject.parseObject(positionJson);
- this.config.setNextPosition(jsonObject.getLong(RmqConstants.NEXT_POSITION));
- } else {
- this.config.setNextPosition(0L);
- }
- mqOffsetMap.put(tti, this.config.getNextPosition());
- }
- }
- } else {
- // divide task by topic
- for (MessageQueue mq: mqs) {
+ for (TaskTopicInfo tti : topicList) {
+ Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(tti.getTopic());
+ for (MessageQueue mq : mqs) {
+ if (Integer.valueOf(tti.getQueueId()) == mq.getQueueId()) {
ByteBuffer positionInfo = this.context.positionStorageReader().getPosition(
- ByteBuffer.wrap(RmqConstants.getPartition(
- mq.getTopic(),
- mq.getBrokerName(),
- String.valueOf(mq.getQueueId())).getBytes("UTF-8")));
+ ByteBuffer.wrap(RmqConstants.getPartition(
+ mq.getTopic(),
+ mq.getBrokerName(),
+ String.valueOf(mq.getQueueId())).getBytes("UTF-8")));
if (null != positionInfo && positionInfo.array().length > 0) {
String positionJson = new String(positionInfo.array(), "UTF-8");
@@ -149,9 +129,8 @@ public class RmqSourceTask extends SourceTask {
if (started) {
try {
for (TaskTopicInfo taskTopicConfig : this.mqOffsetMap.keySet()) {
- MessageQueue mq = taskTopicConfig.convertMQ();
- PullResult pullResult = consumer.pull(mq, "*",
- this.mqOffsetMap.get(mq), 32);
+ PullResult pullResult = consumer.pull(taskTopicConfig, "*",
+ this.mqOffsetMap.get(taskTopicConfig), 32);
switch (pullResult.getPullStatus()) {
case FOUND: {
this.mqOffsetMap.put(taskTopicConfig, pullResult.getNextBeginOffset());
@@ -160,21 +139,21 @@ public class RmqSourceTask extends SourceTask {
List<MessageExt> msgs = pullResult.getMsgFoundList();
Schema schema = new Schema();
schema.setDataSource(this.config.getSourceRocketmq());
- schema.setName(mq.getTopic());
+ schema.setName(taskTopicConfig.getTopic());
schema.setFields(new ArrayList<Field>());
schema.getFields().add(new Field(0,
- FieldName.COMMON_MESSAGE.getKey(), FieldType.STRING));
+ FieldName.COMMON_MESSAGE.getKey(), FieldType.STRING));
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
+ .queue(this.config.getStoreTopic()).entryType(EntryType.CREATE);
dataEntryBuilder.putFiled(FieldName.COMMON_MESSAGE.getKey(), JSONObject.toJSONString(msgs));
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap(RmqConstants.getPartition(
- mq.getTopic(),
- mq.getBrokerName(),
- String.valueOf(mq.getQueueId())).getBytes("UTF-8")),
- ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8"))
+ ByteBuffer.wrap(RmqConstants.getPartition(
+ taskTopicConfig.getTopic(),
+ taskTopicConfig.getBrokerName(),
+ String.valueOf(taskTopicConfig.getQueueId())).getBytes("UTF-8")),
+ ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8"))
);
sourceDataEntry.setQueueName(taskTopicConfig.getTargetTopic());
res.add(sourceDataEntry);
diff --git a/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
index c5a39e4..1086295 100644
--- a/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
+++ b/src/main/java/org/apache/rocketmq/replicator/config/TaskTopicInfo.java
@@ -18,44 +18,15 @@ package org.apache.rocketmq.replicator.config;
import org.apache.rocketmq.common.message.MessageQueue;
-public class TaskTopicInfo {
+public class TaskTopicInfo extends MessageQueue{
- private String sourceTopic;
- private String brokerName;
- private String queueId;
private String targetTopic;
- public TaskTopicInfo(String sourceTopic, String brokerName, String queueId, String targetTopic) {
- this.sourceTopic = sourceTopic;
- this.brokerName = brokerName;
- this.queueId = queueId;
+ public TaskTopicInfo(String sourceTopic, String brokerName, int queueId, String targetTopic) {
+ super(sourceTopic, brokerName, queueId);
this.targetTopic = targetTopic;
}
- public String getSourceTopic() {
- return sourceTopic;
- }
-
- public void setSourceTopic(String sourceTopic) {
- this.sourceTopic = sourceTopic;
- }
-
- public String getBrokerName() {
- return brokerName;
- }
-
- public void setBrokerName(String brokerName) {
- this.brokerName = brokerName;
- }
-
- public String getQueueId() {
- return queueId;
- }
-
- public void setQueueId(String queueId) {
- this.queueId = queueId;
- }
-
public String getTargetTopic() {
return this.targetTopic;
}
@@ -63,9 +34,4 @@ public class TaskTopicInfo {
public void setTargetTopic(String targetTopic) {
this.targetTopic = targetTopic;
}
-
- public MessageQueue convertMQ() {
- return new MessageQueue(sourceTopic,
- brokerName, Integer.parseInt(queueId));
- }
}
[rocketmq-connect] 29/39: [ISSUE 503] Metadata synchronization optimization (#504)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit ae2751eb5d72a643846414f65b7512186f4077ce
Author: zhoubo <87...@qq.com>
AuthorDate: Thu Dec 26 20:54:00 2019 +0800
[ISSUE 503] Metadata synchronization optimization (#504)
* TopicList is null exception and frequent requestTaskReconfiguration
* https://github.com/apache/rocketmq-externals/issues/478
* * Runtime add some log
* Fix replicator add new topic frequent requestTaskReconfiguration bug
* Optimize metadata synchronization and fix RocketMQConverter bug
https://github.com/apache/rocketmq-externals/issues/492
* Fix replicator bug
* Exclude groups starting with RebalanceService
* 1.Metadata synchronization bug fix
2.consumerGroup filters out the ones used in runtime
3.Multiple task consumerGroup name duplication issues
4.Incorrect offsetSyncTopic assignment in task configuration
---
.../java/org/apache/rocketmq/replicator/RmqMetaReplicator.java | 8 +++++---
src/main/java/org/apache/rocketmq/replicator/common/Utils.java | 6 +++---
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
index 75bc919..856fce8 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqMetaReplicator.java
@@ -183,12 +183,14 @@ public class RmqMetaReplicator extends SourceConnector {
Collections.shuffle(masters);
Set<String> targetBrokers =
- CommandUtil.fetchMasterAddrByClusterName(this.targetMQAdminExt, replicatorConfig.getSrcCluster());
+ CommandUtil.fetchMasterAddrByClusterName(this.targetMQAdminExt, replicatorConfig.getTargetCluster());
String addr = masters.get(0);
SubscriptionGroupWrapper sub = this.srcMQAdminExt.getAllSubscriptionGroup(addr, TimeUnit.SECONDS.toMillis(10));
for (Map.Entry<String, SubscriptionGroupConfig> entry : sub.getSubscriptionGroupTable().entrySet()) {
- ensureSubConfig(targetBrokers, entry.getValue());
+ if (skipInnerGroup(entry.getKey())) {
+ ensureSubConfig(targetBrokers, entry.getValue());
+ }
}
} catch (Exception e) {
log.error("syncSubConfig failed", e);
@@ -196,7 +198,7 @@ public class RmqMetaReplicator extends SourceConnector {
}
private void ensureSubConfig(Collection<String> targetBrokers,
- SubscriptionGroupConfig subConfig) throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
+ SubscriptionGroupConfig subConfig) throws InterruptedException, RemotingException, MQClientException, MQBrokerException {
for (String addr : targetBrokers) {
this.targetMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, subConfig);
}
diff --git a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
index 60687d7..9ccee41 100644
--- a/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
+++ b/src/main/java/org/apache/rocketmq/replicator/common/Utils.java
@@ -22,8 +22,8 @@ import io.openmessaging.internal.DefaultKeyValue;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -44,7 +44,7 @@ public class Utils {
private static final Logger log = LoggerFactory.getLogger(Utils.class);
public static String createGroupName(String prefix) {
- return new StringBuilder().append(prefix).append("-").append(System.currentTimeMillis()).toString();
+ return new StringBuilder().append(prefix).append("-").append(System.currentTimeMillis()).append("-").append(ThreadLocalRandom.current().nextInt()).toString();
}
public static String createGroupName(String prefix, String postfix) {
@@ -132,7 +132,7 @@ public class Utils {
keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSrcNamesrvs());
keyValue.put(TaskConfigEnum.TASK_SOURCE_CLUSTER.getKey(), tdc.getSrcCluster());
- keyValue.put(TaskConfigEnum.TASK_OFFSET_SYNC_TOPIC.getKey(), tdc.getSrcCluster());
+ keyValue.put(TaskConfigEnum.TASK_OFFSET_SYNC_TOPIC.getKey(), tdc.getOffsetSyncTopic());
keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.OFFSET.ordinal());
keyValue.put(TaskConfigEnum.TASK_GROUP_INFO.getKey(), JSONObject.toJSONString(groupList));
keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getConverter());
[rocketmq-connect] 30/39: Supplemental replicator documentation (#506)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit d2e168e61a03a2d5b72bfe954afb49be810a3049
Author: zhoubo <87...@qq.com>
AuthorDate: Sun Dec 29 21:48:44 2019 +0800
Supplemental replicator documentation (#506)
---
README.md | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/README.md b/README.md
index 538568b..0cb4c9a 100644
--- a/README.md
+++ b/README.md
@@ -3,8 +3,11 @@
## rocketmq-replicator打包
````
mvn clean install -Prelease-all -DskipTest -U
+打包成功后将rocketmq-replicator-0.1.0-SNAPSHOT-jar-with-dependencies.jar（fatjar）放到runtime配置的pluginPaths目录下
````
## rocketmq-replicator启动
+
+同步topic和消息
````
http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}
?config={"connector-class":"org.apache.rocketmq.replicator.RmqSourceReplicator","source-rocketmq":"xxxx:9876","target-rocketmq":"xxxxxxx:9876","replicator-store-topic":"replicatorTopic","taskDivideStrategy":"0","white-list":"TopicTest,TopicTest2","task-parallelism":"2","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
@@ -17,6 +20,10 @@ http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}/stop
````
## rocketmq-meta-connector启动
+
+同步消费消费进度和ConsumerGroup
+
+注：此功能尚不成熟还需要后续版本优化
````
http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-replicator-name}
?config={"connector-class":"org.apache.rocketmq.replicator.RmqMetaReplicator","source-rocketmq":"xxxx:9876","target-rocketmq":"xxxxxxx:9876","replicator-store-topic":"replicatorTopic","offset.sync.topic":"syncTopic","taskDivideStrategy":"0","white-list":"TopicTest,TopicTest2","task-parallelism":"2","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
@@ -34,8 +41,11 @@ parameter | type | must | description | sample value
---|---|---|---|---|
source-rocketmq | String | Yes | namesrv address of source rocketmq cluster | 192.168.1.2:9876 |
target-rocketmq | String | Yes | namesrv address of target rocketmq cluster | 192.168.1.2:9876 |
+target-cluster | String | Yes | target rocketmq cluster name | DefaultCluster |
+source-cluster | String | Yes | source rocketmq cluster name | DefaultCluster |
replicator-store-topic | String | Yes | topic name to store all source messages | replicator-store-topic |
task-divide-strategy | Integer | No | task dividing strategy, default value is 0 for dividing by topic | 0 |
white-list | String | Yes | topic white list and multiple fields are separated by commas | topic-1,topic-2 |
task-parallelism | String | No | task parallelism，default value is 1，one task will be responsible for multiple topics for the value greater than 1 | 2 |
source-record-converter | String | Yes | source data parser | io.openmessaging.connect.runtime.converter.JsonConverter |
+topic.rename.format | String | Yes | rename topic name rules | rename-${topic} (${topic} represents the source topic name) |
[rocketmq-connect] 14/39: fix sourceTaskTopicList is null. resolve #386
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit d4e677dc7dda43ae4e42b09936735bfc3175820b
Author: xujianhai666 <ze...@bytedance.com>
AuthorDate: Fri Sep 20 22:36:10 2019 +0800
fix sourceTaskTopicList is null. resolve #386
---
src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
index b16e585..42dbab8 100644
--- a/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/replicator/RmqSourceTask.java
@@ -76,6 +76,10 @@ public class RmqSourceTask extends SourceTask {
List<TaskTopicInfo> topicList = JSONObject.parseArray(this.config.getTaskTopicList(), TaskTopicInfo.class);
try {
+ if (topicList == null) {
+ throw new IllegalStateException("topicList is null");
+ }
+
this.consumer.start();
for (TaskTopicInfo tti : topicList) {
Set<MessageQueue> mqs = consumer.fetchSubscribeMessageQueues(tti.getTopic());
[rocketmq-connect] 23/39: feat(replicator): Add commitRecord after producer send success (#452)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit ff09b427a2ffff360381480b42e49fb7f18f9ee8
Author: xujianhai666 <52...@users.noreply.github.com>
AuthorDate: Thu Nov 28 09:10:13 2019 +0800
feat(replicator): Add commitRecord after producer send success (#452)
- Add CommitRecord call after producer send success.
Closes #450
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index dee5b08..fc35065 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,7 @@
<dependency>
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-connector</artifactId>
- <version>0.1.0-beta</version>
+ <version>0.1.1-beta-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>