You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by zh...@apache.org on 2022/03/02 03:22:23 UTC
[rocketmq-connect] branch master updated (00d60fb -> 232c585)
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git.
from 00d60fb Add 'connector/rocketmq-connect-kafka/' from commit '59ac9fd7b7fd11ad7943b5ef30a3f95b20acadd7'
new f629088 add junit test and modify some code
new 6ca400a add junit test and modify some code
new e84812e 修改stop
new 32d3165 fix some bug
new 70caed3 fix some bug
new 6777f23 fix bug
new f95bf22 support multiple mongo replicaset
new 98da0fb fix test verifyConfig
new f193fcd reformat code and add more test case
new 2cdb27a fix some bug and add readme
new db51f7f clean code
new b307466 bump up the connect version to 0.1.1
new 232c585 Add 'connector/rocketmq-connect-mongo/' from commit 'b307466f3fc4fa25ba21de87c568a4095cf18f4f'
The 13 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
connector/rocketmq-connect-mongo/README.md | 43 ++++
.../rocketmq-connect-mongo}/pom.xml | 102 ++++----
.../org/apache/connect/mongo/SourceTaskConfig.java | 266 +++++++++++++++++++++
.../mongo/connector/MongoSourceConnector.java} | 28 ++-
.../connect/mongo/connector/MongoSourceTask.java | 105 ++++++++
.../mongo/connector/builder/MongoDataEntry.java | 134 +++++++++++
.../connect/mongo/initsync/CollectionMeta.java | 58 +++++
.../apache/connect/mongo/initsync/InitSync.java | 163 +++++++++++++
.../connect/mongo/replicator/Constants.java} | 27 +--
.../mongo/replicator/MongoClientFactory.java | 129 ++++++++++
.../connect/mongo/replicator/OperationFilter.java | 80 +++++++
.../apache/connect/mongo/replicator/Position.java | 85 +++++++
.../connect/mongo/replicator/ReplicaSet.java | 112 +++++++++
.../connect/mongo/replicator/ReplicaSetConfig.java | 75 ++++++
.../mongo/replicator/ReplicaSetManager.java | 85 +++++++
.../mongo/replicator/ReplicaSetsContext.java | 121 ++++++++++
.../connect/mongo/replicator/ReplicatorTask.java | 128 ++++++++++
.../replicator/event/Document2EventConverter.java | 49 ++++
.../mongo/replicator/event/OperationType.java | 62 +++++
.../mongo/replicator/event/ReplicationEvent.java | 176 ++++++++++++++
.../java/org/apache/connect/mongo/FilterTest.java | 67 ++++++
.../org/apache/connect/mongo/MongoFactoryTest.java | 177 ++++++++++++++
.../connect/mongo/MongoSourceConnectorTest.java | 95 ++++++++
.../apache/connect/mongo/MongoSourceTaskTest.java | 142 +++++++++++
.../java/org/apache/connect/mongo/MongoTest.java | 138 +++++++++++
.../apache/connect/mongo/OperationTypeTest.java} | 28 +--
.../apache/connect/mongo/ReplicaContextTest.java | 33 +++
.../connect/mongo/ReplicaSetManagerTest.java | 58 +++++
.../org/apache/connect/mongo/ReplicaSetTest.java | 59 +++++
29 files changed, 2728 insertions(+), 97 deletions(-)
create mode 100644 connector/rocketmq-connect-mongo/README.md
copy {rocketmq-connect-cli => connector/rocketmq-connect-mongo}/pom.xml (72%)
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
copy connector/{rocketmq-connect-activemq/src/main/java/org/apache/rocketmq/connect/activemq/connector/ActivemqSourceConnector.java => rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java} (70%)
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/schema/column/DefaultColumnParser.java => rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/Constants.java} (56%)
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/OperationFilter.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/Position.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/Document2EventConverter.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
create mode 100644 connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
create mode 100644 connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/FilterTest.java
create mode 100644 connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
create mode 100644 connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
create mode 100644 connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
create mode 100644 connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoTest.java
copy connector/{rocketmq-connect-kafka/src/test/java/org/apache/rocketmq/connect/kafka/connector/KafkaSourceTaskTest.java => rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/OperationTypeTest.java} (52%)
create mode 100644 connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
create mode 100644 connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaSetManagerTest.java
create mode 100644 connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaSetTest.java
[rocketmq-connect] 13/13: Add 'connector/rocketmq-connect-mongo/' from commit 'b307466f3fc4fa25ba21de87c568a4095cf18f4f'
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 232c585fdee7e6a0ab6d7aef04140019bc9a3be7
Merge: 00d60fb b307466
Author: odbozhou <87...@qq.com>
AuthorDate: Wed Mar 2 11:21:49 2022 +0800
Add 'connector/rocketmq-connect-mongo/' from commit 'b307466f3fc4fa25ba21de87c568a4095cf18f4f'
git-subtree-dir: connector/rocketmq-connect-mongo
git-subtree-mainline: 00d60fb18f82dd396d1ebc0423cde95f32f8fa9e
git-subtree-split: b307466f3fc4fa25ba21de87c568a4095cf18f4f
connector/rocketmq-connect-mongo/README.md | 43 ++++
connector/rocketmq-connect-mongo/pom.xml | 180 ++++++++++++++
.../org/apache/connect/mongo/SourceTaskConfig.java | 266 +++++++++++++++++++++
.../mongo/connector/MongoSourceConnector.java | 76 ++++++
.../connect/mongo/connector/MongoSourceTask.java | 105 ++++++++
.../mongo/connector/builder/MongoDataEntry.java | 134 +++++++++++
.../connect/mongo/initsync/CollectionMeta.java | 58 +++++
.../apache/connect/mongo/initsync/InitSync.java | 163 +++++++++++++
.../apache/connect/mongo/replicator/Constants.java | 36 +++
.../mongo/replicator/MongoClientFactory.java | 129 ++++++++++
.../connect/mongo/replicator/OperationFilter.java | 80 +++++++
.../apache/connect/mongo/replicator/Position.java | 85 +++++++
.../connect/mongo/replicator/ReplicaSet.java | 112 +++++++++
.../connect/mongo/replicator/ReplicaSetConfig.java | 75 ++++++
.../mongo/replicator/ReplicaSetManager.java | 85 +++++++
.../mongo/replicator/ReplicaSetsContext.java | 121 ++++++++++
.../connect/mongo/replicator/ReplicatorTask.java | 128 ++++++++++
.../replicator/event/Document2EventConverter.java | 49 ++++
.../mongo/replicator/event/OperationType.java | 62 +++++
.../mongo/replicator/event/ReplicationEvent.java | 176 ++++++++++++++
.../java/org/apache/connect/mongo/FilterTest.java | 67 ++++++
.../org/apache/connect/mongo/MongoFactoryTest.java | 177 ++++++++++++++
.../connect/mongo/MongoSourceConnectorTest.java | 95 ++++++++
.../apache/connect/mongo/MongoSourceTaskTest.java | 142 +++++++++++
.../java/org/apache/connect/mongo/MongoTest.java | 138 +++++++++++
.../apache/connect/mongo/OperationTypeTest.java | 37 +++
.../apache/connect/mongo/ReplicaContextTest.java | 33 +++
.../connect/mongo/ReplicaSetManagerTest.java | 58 +++++
.../org/apache/connect/mongo/ReplicaSetTest.java | 59 +++++
29 files changed, 2969 insertions(+)
diff --cc connector/rocketmq-connect-mongo/README.md
index 0000000,bec30a3..bec30a3
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/README.md
+++ b/connector/rocketmq-connect-mongo/README.md
diff --cc connector/rocketmq-connect-mongo/pom.xml
index 0000000,5c412bf..5c412bf
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/pom.xml
+++ b/connector/rocketmq-connect-mongo/pom.xml
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
index 0000000,d184b5c..d184b5c
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
index 0000000,5be2e0d..5be2e0d
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
index 0000000,49bcf49..49bcf49
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
index 0000000,1d6dfe5..1d6dfe5
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
index 0000000,4af5060..4af5060
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
index 0000000,a51b727..a51b727
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/Constants.java
index 0000000,7ba1ac4..7ba1ac4
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/Constants.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/Constants.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
index 0000000,11bca8f..11bca8f
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/OperationFilter.java
index 0000000,a173f6c..a173f6c
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/OperationFilter.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/OperationFilter.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/Position.java
index 0000000,29fd856..29fd856
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/Position.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/Position.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
index 0000000,8393316..8393316
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
index 0000000,ced90b8..ced90b8
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
index 0000000,88097d8..88097d8
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
index 0000000,8dd85d7..8dd85d7
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
index 0000000,cd78f24..cd78f24
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/Document2EventConverter.java
index 0000000,99ab707..99ab707
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/Document2EventConverter.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/Document2EventConverter.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
index 0000000,b418666..b418666
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
diff --cc connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
index 0000000,7adca71..7adca71
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
+++ b/connector/rocketmq-connect-mongo/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/FilterTest.java
index 0000000,d5deefd..d5deefd
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/FilterTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/FilterTest.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
index 0000000,e47d2c4..e47d2c4
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
index 0000000,cc02fbc..cc02fbc
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
index 0000000,4983a66..4983a66
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoTest.java
index 0000000,3d900fa..3d900fa
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/MongoTest.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/OperationTypeTest.java
index 0000000,d8c5a9b..d8c5a9b
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/OperationTypeTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/OperationTypeTest.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
index 0000000,16cb959..16cb959
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaSetManagerTest.java
index 0000000,1d3b743..1d3b743
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaSetManagerTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaSetManagerTest.java
diff --cc connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaSetTest.java
index 0000000,07eefae..07eefae
mode 000000,100644..100644
--- a/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaSetTest.java
+++ b/connector/rocketmq-connect-mongo/src/test/java/org/apache/connect/mongo/ReplicaSetTest.java
[rocketmq-connect] 10/13: fix some bug and add readme
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 2cdb27a2b2e418a1228c0d83adf6acff64e79554
Author: 李平 <17...@qq.com>
AuthorDate: Thu Aug 22 15:10:57 2019 +0800
fix some bug and add readme
---
README.md | 22 +++
.../org/apache/connect/mongo/SourceTaskConfig.java | 188 +++++++++++----------
.../mongo/connector/MongoSourceConnector.java | 17 ++
.../connect/mongo/connector/MongoSourceTask.java | 49 +++---
.../mongo/connector/builder/MongoDataEntry.java | 32 +++-
.../connect/mongo/initsync/CollectionMeta.java | 17 ++
.../apache/connect/mongo/initsync/InitSync.java | 27 ++-
.../apache/connect/mongo/replicator/Constants.java | 23 ++-
.../apache/connect/mongo/replicator/Filter.java | 19 ++-
.../mongo/replicator/MongoClientFactory.java | 38 +++--
.../apache/connect/mongo/replicator/Position.java | 85 ++++++++++
.../connect/mongo/replicator/ReplicaSet.java | 32 +++-
.../connect/mongo/replicator/ReplicaSetConfig.java | 93 ++--------
.../{ReplicaSets.java => ReplicaSetManager.java} | 29 +++-
.../mongo/replicator/ReplicaSetsContext.java | 29 +++-
.../connect/mongo/replicator/ReplicatorTask.java | 62 +++++--
...Converter.java => Document2EventConverter.java} | 27 ++-
.../mongo/replicator/event/OperationType.java | 54 ++++--
.../mongo/replicator/event/ReplicationEvent.java | 17 ++
.../java/org/apache/connect/mongo/FilterTest.java | 3 +-
.../org/apache/connect/mongo/MongoFactoryTest.java | 22 +--
.../connect/mongo/MongoSourceConnectorTest.java | 3 +-
.../apache/connect/mongo/MongoSourceTaskTest.java | 3 +-
.../java/org/apache/connect/mongo/MongoTest.java | 25 ++-
.../apache/connect/mongo/OperationTypeTest.java | 37 ++++
...icaSetsTest.java => ReplicaSetManagerTest.java} | 20 +--
26 files changed, 687 insertions(+), 286 deletions(-)
diff --git a/README.md b/README.md
index fb2c6c6..e32c455 100644
--- a/README.md
+++ b/README.md
@@ -10,3 +10,25 @@ and then init a mongo replicaSet
`docker exec -it mongo-test mongo ` and `rs.initiate()` and then you can run all junit test
+
+
+## task config params
+
+| param | Description | type |
+| --- | --- | --- |
+| mongoAddr | shardName=replicaSetName/127.0.0.1:2781,127.0.0.1:2782,127.0.0.1:2783; | string, split by ; |
+| mongoUserName | mongo root username| string |
+| mongoPassWord | mongo root password| string |
+| interestDbAndCollection | {"dbName":["collection1","collection2"]}, collectionName can be "*" means all collection | json |
+| positionTimeStamp | mongo oplog `bsontimestamp.value`, runtime store position is highest level | int |
+| positionInc | mongo oplog `bsontimestamp.inc`, runtime store position is highest level | int |
+| dataSync | sync all interestDbAndCollection data, runtime store position is highest level | json, Map<String(dbName), List<String(collectionName)>> |
+| serverSelectionTimeoutMS | mongo driver select replicaServer timeout | long |
+| connectTimeoutMS | mongo driver connect socket timeout | long |
+| socketTimeoutMS | mongo driver read or write timeout | long |
+| ssl or tsl | mongo driver use ssl or tsl | boolean |
+| tlsInsecure or sslInvalidHostNameAllowed | mongo driver when use ssl or tsl allow invalid hostname | boolean|
+| compressors | compressors way | string (zlib or snappy)
+| zlibCompressionLevel | zlib compressors level| int (1-7)|
+| trustStore | ssl pem| path|
+| trustStorePassword | ssl pem decrypt password | string|
diff --git a/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java b/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
index 3df9dc4..d184b5c 100644
--- a/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
+++ b/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo;
import io.openmessaging.KeyValue;
@@ -9,22 +26,21 @@ import org.bson.BsonTimestamp;
public class SourceTaskConfig {
- private String replicaSet;
private String mongoAddr;
private String mongoUserName;
private String mongoPassWord;
private String interestDbAndCollection;
- private String positionTimeStamp;
- private String positionInc;
- private String dataSync;
- private String serverSelectionTimeoutMS;
- private String connectTimeoutMS;
- private String socketTimeoutMS;
- private String ssl;
- private String tsl;
- private String tlsInsecure;
- private String sslInvalidHostNameAllowed;
- private String tlsAllowInvalidHostnames;
+ private int positionTimeStamp;
+ private int positionInc;
+ private boolean dataSync;
+ private long serverSelectionTimeoutMS;
+ private long connectTimeoutMS;
+ private long socketTimeoutMS;
+ private boolean ssl;
+ private boolean tsl;
+ private boolean tlsInsecure;
+ private boolean sslInvalidHostNameAllowed;
+ private boolean tlsAllowInvalidHostnames;
private String compressors;
private String zlibCompressionLevel;
private String trustStore;
@@ -37,52 +53,28 @@ public class SourceTaskConfig {
}
});
- public String getTrustStore() {
- return trustStore;
- }
-
- public void setTrustStore(String trustStore) {
- this.trustStore = trustStore;
- }
-
- public String getTrustStorePassword() {
- return trustStorePassword;
- }
-
- public void setTrustStorePassword(String trustStorePassword) {
- this.trustStorePassword = trustStorePassword;
- }
-
- public String getZlibCompressionLevel() {
- return zlibCompressionLevel;
- }
-
- public void setZlibCompressionLevel(String zlibCompressionLevel) {
- this.zlibCompressionLevel = zlibCompressionLevel;
- }
-
- public String getPositionInc() {
- return positionInc;
+ public String getMongoAddr() {
+ return mongoAddr;
}
- public void setPositionInc(String positionInc) {
- this.positionInc = positionInc;
+ public void setMongoAddr(String mongoAddr) {
+ this.mongoAddr = mongoAddr;
}
- public int getCopyThread() {
- return copyThread;
+ public String getMongoUserName() {
+ return mongoUserName;
}
- public void setCopyThread(int copyThread) {
- this.copyThread = copyThread;
+ public void setMongoUserName(String mongoUserName) {
+ this.mongoUserName = mongoUserName;
}
- public String getPositionTimeStamp() {
- return positionTimeStamp;
+ public String getMongoPassWord() {
+ return mongoPassWord;
}
- public void setPositionTimeStamp(String positionTimeStamp) {
- this.positionTimeStamp = positionTimeStamp;
+ public void setMongoPassWord(String mongoPassWord) {
+ this.mongoPassWord = mongoPassWord;
}
public String getInterestDbAndCollection() {
@@ -93,107 +85,91 @@ public class SourceTaskConfig {
this.interestDbAndCollection = interestDbAndCollection;
}
- public String getMongoAddr() {
- return mongoAddr;
- }
-
- public void setMongoAddr(String mongoAddr) {
- this.mongoAddr = mongoAddr;
- }
-
- public String getMongoUserName() {
- return mongoUserName;
+ public int getPositionTimeStamp() {
+ return positionTimeStamp;
}
- public void setMongoUserName(String mongoUserName) {
- this.mongoUserName = mongoUserName;
+ public void setPositionTimeStamp(int positionTimeStamp) {
+ this.positionTimeStamp = positionTimeStamp;
}
- public String getMongoPassWord() {
- return mongoPassWord;
+ public int getPositionInc() {
+ return positionInc;
}
- public void setMongoPassWord(String mongoPassWord) {
- this.mongoPassWord = mongoPassWord;
+ public void setPositionInc(int positionInc) {
+ this.positionInc = positionInc;
}
- public String getDataSync() {
+ public boolean isDataSync() {
return dataSync;
}
- public void setDataSync(String dataSync) {
+ public void setDataSync(boolean dataSync) {
this.dataSync = dataSync;
}
- public String getReplicaSet() {
- return replicaSet;
- }
-
- public String getServerSelectionTimeoutMS() {
+ public long getServerSelectionTimeoutMS() {
return serverSelectionTimeoutMS;
}
- public void setServerSelectionTimeoutMS(String serverSelectionTimeoutMS) {
+ public void setServerSelectionTimeoutMS(long serverSelectionTimeoutMS) {
this.serverSelectionTimeoutMS = serverSelectionTimeoutMS;
}
- public void setReplicaSet(String replicaSet) {
- this.replicaSet = replicaSet;
- }
-
- public String getConnectTimeoutMS() {
+ public long getConnectTimeoutMS() {
return connectTimeoutMS;
}
- public void setConnectTimeoutMS(String connectTimeoutMS) {
+ public void setConnectTimeoutMS(long connectTimeoutMS) {
this.connectTimeoutMS = connectTimeoutMS;
}
- public String getSocketTimeoutMS() {
+ public long getSocketTimeoutMS() {
return socketTimeoutMS;
}
- public void setSocketTimeoutMS(String socketTimeoutMS) {
+ public void setSocketTimeoutMS(long socketTimeoutMS) {
this.socketTimeoutMS = socketTimeoutMS;
}
- public String getSsl() {
+ public boolean getSsl() {
return ssl;
}
- public void setSsl(String ssl) {
+ public void setSsl(boolean ssl) {
this.ssl = ssl;
}
- public String getTsl() {
+ public boolean getTsl() {
return tsl;
}
- public void setTsl(String tsl) {
+ public void setTsl(boolean tsl) {
this.tsl = tsl;
}
- public String getTlsInsecure() {
+ public boolean getTlsInsecure() {
return tlsInsecure;
}
- public void setTlsInsecure(String tlsInsecure) {
+ public void setTlsInsecure(boolean tlsInsecure) {
this.tlsInsecure = tlsInsecure;
}
- public String getSslInvalidHostNameAllowed() {
+ public boolean getSslInvalidHostNameAllowed() {
return sslInvalidHostNameAllowed;
}
- public void setSslInvalidHostNameAllowed(String sslInvalidHostNameAllowed) {
+ public void setSslInvalidHostNameAllowed(boolean sslInvalidHostNameAllowed) {
this.sslInvalidHostNameAllowed = sslInvalidHostNameAllowed;
}
- public String getTlsAllowInvalidHostnames() {
+ public boolean getTlsAllowInvalidHostnames() {
return tlsAllowInvalidHostnames;
}
- public void setTlsAllowInvalidHostnames(String tlsAllowInvalidHostnames) {
+ public void setTlsAllowInvalidHostnames(boolean tlsAllowInvalidHostnames) {
this.tlsAllowInvalidHostnames = tlsAllowInvalidHostnames;
}
@@ -205,6 +181,38 @@ public class SourceTaskConfig {
this.compressors = compressors;
}
+ public String getZlibCompressionLevel() {
+ return zlibCompressionLevel;
+ }
+
+ public void setZlibCompressionLevel(String zlibCompressionLevel) {
+ this.zlibCompressionLevel = zlibCompressionLevel;
+ }
+
+ public String getTrustStore() {
+ return trustStore;
+ }
+
+ public void setTrustStore(String trustStore) {
+ this.trustStore = trustStore;
+ }
+
+ public String getTrustStorePassword() {
+ return trustStorePassword;
+ }
+
+ public void setTrustStorePassword(String trustStorePassword) {
+ this.trustStorePassword = trustStorePassword;
+ }
+
+ public int getCopyThread() {
+ return copyThread;
+ }
+
+ public void setCopyThread(int copyThread) {
+ this.copyThread = copyThread;
+ }
+
public void load(KeyValue props) {
properties2Object(props, this);
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
index e3dfb6f..5be2e0d 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.connector;
import io.openmessaging.KeyValue;
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
index da244cd..49bcf49 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.connector;
import com.alibaba.fastjson.JSONObject;
@@ -7,13 +24,10 @@ import io.openmessaging.connector.api.source.SourceTask;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
-import java.util.regex.Pattern;
-import org.apache.commons.lang3.StringUtils;
import org.apache.connect.mongo.SourceTaskConfig;
-import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.replicator.Position;
import org.apache.connect.mongo.replicator.ReplicaSet;
-import org.apache.connect.mongo.replicator.ReplicaSetConfig;
-import org.apache.connect.mongo.replicator.ReplicaSets;
+import org.apache.connect.mongo.replicator.ReplicaSetManager;
import org.apache.connect.mongo.replicator.ReplicaSetsContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -24,12 +38,10 @@ public class MongoSourceTask extends SourceTask {
private SourceTaskConfig sourceTaskConfig;
- private ReplicaSets replicaSets;
+ private ReplicaSetManager replicaSetManager;
private ReplicaSetsContext replicaSetsContext;
- private Pattern pattern = Pattern.compile("^[-\\+]?[\\d]*$");
-
@Override
public Collection<SourceDataEntry> poll() {
@@ -41,24 +53,23 @@ public class MongoSourceTask extends SourceTask {
try {
sourceTaskConfig = new SourceTaskConfig();
sourceTaskConfig.load(config);
+
replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
- replicaSets = ReplicaSets.create(sourceTaskConfig.getMongoAddr());
- replicaSets.getReplicaConfigByName().forEach((replicaSetName, replicaSetConfig) -> {
+
+ replicaSetManager = ReplicaSetManager.create(sourceTaskConfig.getMongoAddr());
+
+ replicaSetManager.getReplicaConfigByName().forEach((replicaSetName, replicaSetConfig) -> {
ByteBuffer byteBuffer = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(
replicaSetName.getBytes()));
if (byteBuffer != null && byteBuffer.array().length > 0) {
String positionJson = new String(byteBuffer.array(), StandardCharsets.UTF_8);
- ReplicaSetConfig.Position position = JSONObject.parseObject(positionJson, ReplicaSetConfig.Position.class);
+ Position position = JSONObject.parseObject(positionJson, Position.class);
replicaSetConfig.setPosition(position);
} else {
- ReplicaSetConfig.Position position = replicaSetConfig.emptyPosition();
- position.setTimeStamp(sourceTaskConfig.getPositionTimeStamp() != null
- && pattern.matcher(sourceTaskConfig.getPositionTimeStamp()).matches()
- ? Integer.valueOf(sourceTaskConfig.getPositionTimeStamp()) : 0);
- position.setInc(sourceTaskConfig.getPositionInc() != null
- && pattern.matcher(sourceTaskConfig.getPositionInc()).matches()
- ? Integer.valueOf(sourceTaskConfig.getPositionInc()) : 0);
- position.setInitSync(StringUtils.equals(sourceTaskConfig.getDataSync(), Constants.INITSYNC) ? true : false);
+ Position position = new Position();
+ position.setTimeStamp(sourceTaskConfig.getPositionTimeStamp());
+ position.setInc(sourceTaskConfig.getPositionInc());
+ position.setInitSync(sourceTaskConfig.isDataSync());
replicaSetConfig.setPosition(position);
}
diff --git a/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java b/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
index 87d92ea..1d6dfe5 100644
--- a/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
+++ b/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.connector.builder;
import com.alibaba.fastjson.JSONObject;
@@ -10,6 +27,7 @@ import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.replicator.Position;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
@@ -17,8 +35,8 @@ import org.bson.BsonTimestamp;
import static org.apache.connect.mongo.replicator.Constants.CREATED;
import static org.apache.connect.mongo.replicator.Constants.NAMESPACE;
-import static org.apache.connect.mongo.replicator.Constants.OBJECTID;
-import static org.apache.connect.mongo.replicator.Constants.OPERATIONTYPE;
+import static org.apache.connect.mongo.replicator.Constants.OBJECT_ID;
+import static org.apache.connect.mongo.replicator.Constants.OPERATION_TYPE;
import static org.apache.connect.mongo.replicator.Constants.PATCH;
import static org.apache.connect.mongo.replicator.Constants.TIMESTAMP;
import static org.apache.connect.mongo.replicator.Constants.VERSION;
@@ -48,12 +66,12 @@ public class MongoDataEntry {
dataEntryBuilder.timestamp(System.currentTimeMillis())
.queue(event.getNamespace().replace(".", "-").replace("$", "-"))
.entryType(event.getEntryType());
- dataEntryBuilder.putFiled(OPERATIONTYPE, event.getOperationType().name());
+ dataEntryBuilder.putFiled(OPERATION_TYPE, event.getOperationType().name());
dataEntryBuilder.putFiled(TIMESTAMP, event.getTimestamp().getValue());
dataEntryBuilder.putFiled(VERSION, event.getV());
dataEntryBuilder.putFiled(NAMESPACE, event.getNamespace());
dataEntryBuilder.putFiled(PATCH, event.getEventData().isPresent() ? JSONObject.toJSONString(event.getEventData().get()) : "");
- dataEntryBuilder.putFiled(OBJECTID, event.getObjectId().isPresent() ? JSONObject.toJSONString(event.getObjectId().get()) : "");
+ dataEntryBuilder.putFiled(OBJECT_ID, event.getObjectId().isPresent() ? JSONObject.toJSONString(event.getObjectId().get()) : "");
}
String position = createPosition(event, replicaSetConfig);
@@ -64,7 +82,7 @@ public class MongoDataEntry {
}
private static String createPosition(ReplicationEvent event, ReplicaSetConfig replicaSetConfig) {
- ReplicaSetConfig.Position position = replicaSetConfig.emptyPosition();
+ Position position = new Position();
BsonTimestamp timestamp = event.getTimestamp();
position.setInc(timestamp != null ? timestamp.getInc() : 0);
position.setTimeStamp(timestamp != null ? timestamp.getTime() : 0);
@@ -99,7 +117,7 @@ public class MongoDataEntry {
private static void oplogField(Schema schema) {
schema.setFields(new ArrayList<>());
- Field op = new Field(0, OPERATIONTYPE, FieldType.STRING);
+ Field op = new Field(0, OPERATION_TYPE, FieldType.STRING);
schema.getFields().add(op);
Field time = new Field(1, TIMESTAMP, FieldType.INT64);
schema.getFields().add(time);
@@ -109,7 +127,7 @@ public class MongoDataEntry {
schema.getFields().add(namespace);
Field patch = new Field(4, PATCH, FieldType.STRING);
schema.getFields().add(patch);
- Field objectId = new Field(5, OBJECTID, FieldType.STRING);
+ Field objectId = new Field(5, OBJECT_ID, FieldType.STRING);
schema.getFields().add(objectId);
}
diff --git a/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java b/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
index 9c73eac..4af5060 100644
--- a/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
+++ b/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.initsync;
public class CollectionMeta {
diff --git a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
index 6cdbe06..3d68fac 100644
--- a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
+++ b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.initsync;
import com.mongodb.client.MongoClient;
@@ -13,7 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.connect.mongo.replicator.ReplicaSet;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
import org.apache.connect.mongo.replicator.ReplicaSetsContext;
-import org.apache.connect.mongo.replicator.event.EventConverter;
+import org.apache.connect.mongo.replicator.event.Document2EventConverter;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
import org.bson.Document;
@@ -115,24 +132,24 @@ public class InitSync {
.batchSize(200)
.iterator();
while (replicaSet.isRuning() && mongoCursor.hasNext()) {
- if (context.initSyncAbort()) {
+ if (context.isInitSyncAbort()) {
logger.info("init sync database:{}, collection:{} abort, has copy:{} document", collectionMeta.getDatabaseName(), collectionMeta.getCollectionName(), count);
return;
}
count++;
Document document = mongoCursor.next();
- ReplicationEvent event = EventConverter.convert(document, replicaSetConfig.getReplicaSetName());
+ ReplicationEvent event = Document2EventConverter.convert(document, replicaSetConfig.getReplicaSetName());
event.setOperationType(OperationType.CREATED);
event.setNamespace(collectionMeta.getNameSpace());
context.publishEvent(event, replicaSetConfig);
}
} catch (Exception e) {
- context.initSyncError();
+ context.setInitSyncError();
+ replicaSet.shutdown();
logger.error("init sync database:{}, collection:{} error", collectionMeta.getDatabaseName(), collectionMeta.getNameSpace(), e);
} finally {
countDownLatch.countDown();
- replicaSet.shutdown();
}
logger.info("database:{}, collection:{}, copy {} documents, init sync done", collectionMeta.getDatabaseName(), collectionMeta.getCollectionName(), count);
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Constants.java b/src/main/java/org/apache/connect/mongo/replicator/Constants.java
index c895bd6..7ba1ac4 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/Constants.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/Constants.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator;
public class Constants {
@@ -5,17 +22,15 @@ public class Constants {
public static final String MONGO_LOCAL_DATABASE = "local";
public static final String MONGO_OPLOG_RS = "oplog.rs";
- public static final String OPERATIONTYPE = "op";
+ public static final String OPERATION_TYPE = "op";
public static final String TIMESTAMP = "ts";
public static final String VERSION = "v";
public static final String HASH = "h";
public static final String NAMESPACE = "ns";
public static final String OPERATION = "o";
- public static final String OBJECTID = "o2";
+ public static final String OBJECT_ID = "o2";
public static final String CREATED = "created";
public static final String PATCH = "patch";
- public static final String INITSYNC = "initSync";
-
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Filter.java b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
index fd26163..a517822 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/Filter.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator;
import com.alibaba.fastjson.JSONObject;
@@ -49,7 +66,7 @@ public class Filter {
return false;
};
- notNoopFilter = (opeartionType) -> opeartionType.ordinal() != OperationType.NOOP.ordinal();
+ notNoopFilter = (operationType) -> !operationType.equals(OperationType.NOOP);
}
public boolean filterMeta(CollectionMeta collectionMeta) {
diff --git a/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java b/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
index f5e01a3..11bca8f 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator;
import com.mongodb.ConnectionString;
@@ -38,46 +55,46 @@ public class MongoClientFactory {
sb.append(replicaSetConfig.getReplicaSetName());
}
- if (StringUtils.isNotBlank(taskConfig.getServerSelectionTimeoutMS())) {
+ if (taskConfig.getServerSelectionTimeoutMS() > 0) {
sb.append("&");
sb.append("serverSelectionTimeoutMS=");
sb.append(taskConfig.getServerSelectionTimeoutMS());
}
- if (StringUtils.isNotBlank(taskConfig.getConnectTimeoutMS())) {
+ if (taskConfig.getConnectTimeoutMS() > 0) {
sb.append("&");
sb.append("connectTimeoutMS=");
sb.append(taskConfig.getConnectTimeoutMS());
}
- if (StringUtils.isNotBlank(taskConfig.getSocketTimeoutMS())) {
+ if (taskConfig.getSocketTimeoutMS() > 0) {
sb.append("&");
sb.append("socketTimeoutMS=");
sb.append(taskConfig.getSocketTimeoutMS());
}
- if (StringUtils.isNotBlank(taskConfig.getSsl()) || StringUtils.isNotBlank(taskConfig.getTsl())) {
+ if (taskConfig.getSsl() || taskConfig.getTsl()) {
sb.append("&");
sb.append("ssl=");
sb.append(true);
}
- if (StringUtils.isNotBlank(taskConfig.getTlsInsecure())) {
+ if (taskConfig.getTlsInsecure()) {
sb.append("&");
sb.append("tlsInsecure=");
- sb.append(taskConfig.getTlsInsecure());
+ sb.append(true);
}
- if (StringUtils.isNotBlank(taskConfig.getTlsAllowInvalidHostnames())) {
+ if (taskConfig.getTlsAllowInvalidHostnames()) {
sb.append("&");
sb.append("tlsAllowInvalidHostnames=");
- sb.append(taskConfig.getTlsAllowInvalidHostnames());
+ sb.append(true);
}
- if (StringUtils.isNotBlank(taskConfig.getSslInvalidHostNameAllowed())) {
+ if (taskConfig.getSslInvalidHostNameAllowed()) {
sb.append("&");
sb.append("sslInvalidHostNameAllowed=");
- sb.append(taskConfig.getSslInvalidHostNameAllowed());
+ sb.append(true);
}
if (StringUtils.isNotBlank(taskConfig.getCompressors())) {
@@ -105,7 +122,6 @@ public class MongoClientFactory {
}
logger.info("connection string :{}", sb.toString());
- System.out.println(sb.toString());
ConnectionString connectionString = new ConnectionString(sb.toString());
return MongoClients.create(connectionString);
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Position.java b/src/main/java/org/apache/connect/mongo/replicator/Position.java
new file mode 100644
index 0000000..29fd856
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/Position.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.connect.mongo.replicator;
+
+import java.util.Objects;
+import org.bson.BsonTimestamp;
+
+public class Position {
+
+ private int timeStamp;
+ private int inc;
+ private boolean initSync;
+
+ public int getTimeStamp() {
+ return timeStamp;
+ }
+
+ public void setTimeStamp(int timeStamp) {
+ this.timeStamp = timeStamp;
+ }
+
+ public int getInc() {
+ return inc;
+ }
+
+ public void setInc(int inc) {
+ this.inc = inc;
+ }
+
+ public boolean isInitSync() {
+ return initSync;
+ }
+
+ public void setInitSync(boolean initSync) {
+ this.initSync = initSync;
+ }
+
+ public Position() {
+
+ }
+
+ public Position(int timeStamp, int inc, boolean initSync) {
+ this.timeStamp = timeStamp;
+ this.inc = inc;
+ this.initSync = initSync;
+ }
+
+ public boolean isValid() {
+ return timeStamp > 0 && inc > 0;
+ }
+
+ public BsonTimestamp converBsonTimeStamp() {
+ return new BsonTimestamp(timeStamp, inc);
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ Position position = (Position) o;
+ return timeStamp == position.timeStamp &&
+ inc == position.inc &&
+ initSync == position.initSync;
+ }
+
+ @Override public int hashCode() {
+ return Objects.hash(timeStamp, inc, initSync);
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
index 8f4d0d8..8393316 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator;
import com.mongodb.client.MongoClient;
@@ -38,14 +55,14 @@ public class ReplicaSet {
}
public void start() {
+ if (!running.compareAndSet(false, true)) {
+ logger.info("the java mongo replica already start");
+ return;
+ }
try {
- if (!running.compareAndSet(false, true)) {
- logger.info("the java mongo replica already start");
- return;
- }
this.mongoClient = replicaSetsContext.createMongoClient(replicaSetConfig);
- this.isReplicaMongo();
+ this.checkReplicaMongo();
executorService.submit(new ReplicatorTask(this, mongoClient, replicaSetConfig, replicaSetsContext));
} catch (Exception e) {
logger.error("start replicator:{} error", replicaSetConfig, e);
@@ -53,16 +70,15 @@ public class ReplicaSet {
}
}
- public boolean isReplicaMongo() {
+ public void checkReplicaMongo() {
MongoDatabase local = mongoClient.getDatabase(MONGO_LOCAL_DATABASE);
MongoIterable<String> collectionNames = local.listCollectionNames();
MongoCursor<String> iterator = collectionNames.iterator();
while (iterator.hasNext()) {
if (StringUtils.equals(MONGO_OPLOG_RS, iterator.next())) {
- return true;
+ return;
}
}
- this.shutdown();
throw new IllegalStateException(String.format("url:%s, is not replica", replicaSetConfig.getHost()));
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
index 1b54b17..ced90b8 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
@@ -1,7 +1,21 @@
-package org.apache.connect.mongo.replicator;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
-import java.util.Objects;
-import org.bson.BsonTimestamp;
+package org.apache.connect.mongo.replicator;
public class ReplicaSetConfig {
@@ -48,10 +62,6 @@ public class ReplicaSetConfig {
this.host = host;
}
- public Position emptyPosition() {
- return new Position(0, 0, true);
- }
-
@Override
public String toString() {
return "ReplicaSetConfig{" +
@@ -62,73 +72,4 @@ public class ReplicaSetConfig {
'}';
}
- public class Position {
- private int timeStamp;
- private int inc;
- private boolean initSync;
-
- public int getTimeStamp() {
- return timeStamp;
- }
-
- public void setTimeStamp(int timeStamp) {
- this.timeStamp = timeStamp;
- }
-
- public int getInc() {
- return inc;
- }
-
- public void setInc(int inc) {
- this.inc = inc;
- }
-
- public boolean isInitSync() {
- return initSync;
- }
-
- public void setInitSync(boolean initSync) {
- this.initSync = initSync;
- }
-
- public Position(int timeStamp, int inc, boolean initSync) {
- this.timeStamp = timeStamp;
- this.inc = inc;
- this.initSync = initSync;
- }
-
- public boolean isValid() {
- return timeStamp > 0;
- }
-
- public BsonTimestamp converBsonTimeStamp() {
- return new BsonTimestamp(timeStamp, inc);
- }
-
- @Override
- public String toString() {
- return "Position{" +
- "timeStamp=" + timeStamp +
- ", inc=" + inc +
- ", initSync=" + initSync +
- '}';
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
- Position position = (Position) o;
- return timeStamp == position.timeStamp &&
- inc == position.inc &&
- initSync == position.initSync;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(timeStamp, inc, initSync);
- }
- }
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
similarity index 65%
rename from src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java
rename to src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
index af1ebeb..c5757d8 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator;
import java.util.HashMap;
@@ -9,13 +26,15 @@ import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
-public class ReplicaSets {
+public class ReplicaSetManager {
private static final Pattern HOST_PATTERN = Pattern.compile("((([^=]+)[=])?(([^/]+)\\/))?(.+)");
+ private static final String HOST_SEPARATOR = ";";
+
private final Map<String, ReplicaSetConfig> replicaConfigByName = new HashMap<>();
- public ReplicaSets(Set<ReplicaSetConfig> replicaSetConfigs) {
+ public ReplicaSetManager(Set<ReplicaSetConfig> replicaSetConfigs) {
replicaSetConfigs.forEach(replicaSetConfig -> {
if (StringUtils.isNotBlank(replicaSetConfig.getReplicaSetName())) {
replicaConfigByName.put(replicaSetConfig.getReplicaSetName(), replicaSetConfig);
@@ -25,10 +44,10 @@ public class ReplicaSets {
validate();
}
- public static ReplicaSets create(String hosts) {
+ public static ReplicaSetManager create(String hosts) {
Set<ReplicaSetConfig> replicaSetConfigs = new HashSet<>();
if (hosts != null) {
- for (String replicaSetStr : StringUtils.split(hosts.trim(), ";")) {
+ for (String replicaSetStr : StringUtils.split(hosts.trim(), HOST_SEPARATOR)) {
if (StringUtils.isNotBlank(replicaSetStr)) {
ReplicaSetConfig replicaSetConfig = parseReplicaSetStr(replicaSetStr);
if (replicaSetConfig != null) {
@@ -37,7 +56,7 @@ public class ReplicaSets {
}
}
}
- return new ReplicaSets(replicaSetConfigs);
+ return new ReplicaSetManager(replicaSetConfigs);
}
private static ReplicaSetConfig parseReplicaSetStr(String hosts) {
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
index e599f5b..b067256 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
@@ -1,12 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator;
import com.mongodb.client.MongoClient;
import io.openmessaging.connector.api.data.SourceDataEntry;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.connect.mongo.SourceTaskConfig;
@@ -28,9 +46,12 @@ public class ReplicaSetsContext {
private MongoClientFactory mongoClientFactory;
+ private Map<String, Position> lastPositionMap;
+
public ReplicaSetsContext(SourceTaskConfig taskConfig) {
this.taskConfig = taskConfig;
- this.replicaSets = new CopyOnWriteArrayList<>();
+ this.replicaSets = new ArrayList<>();
+ this.lastPositionMap = new HashMap<>();
this.dataEntryQueue = new LinkedBlockingDeque<>();
this.filter = new Filter(taskConfig);
this.mongoClientFactory = new MongoClientFactory(taskConfig);
@@ -91,11 +112,11 @@ public class ReplicaSetsContext {
return res;
}
- public boolean initSyncAbort() {
+ public boolean isInitSyncAbort() {
return initSyncAbort.get();
}
- public void initSyncError() {
+ public void setInitSyncError() {
initSyncAbort.set(true);
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
index 6cb46d1..4c142ce 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator;
import com.mongodb.CursorType;
@@ -7,8 +24,9 @@ import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import org.apache.connect.mongo.initsync.InitSync;
-import org.apache.connect.mongo.replicator.event.EventConverter;
+import org.apache.connect.mongo.replicator.event.Document2EventConverter;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.bson.BsonTimestamp;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,19 +54,27 @@ public class ReplicatorTask implements Runnable {
@Override
public void run() {
- if (replicaSetConfig.getPosition() == null || replicaSetConfig.getPosition().isInitSync()) {
+ BsonTimestamp firstAvailablePosition = findOplogFirstPosition();
+
+ // inValid or
+ // user config dataSync or
+ // user config or runtime saved position lt first oplog position maybe some operation is lost so need dataSync
+ if (!replicaSetConfig.getPosition().isValid() || replicaSetConfig.getPosition().isInitSync()
+ || replicaSetConfig.getPosition().converBsonTimeStamp().compareTo(firstAvailablePosition) < 0) {
+ recordOplogLastPosition();
InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
initSync.start();
+
}
- MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
- FindIterable<Document> iterable;
- if (replicaSetConfig.getPosition().isValid()) {
- iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find(
- Filters.gt("ts", replicaSetConfig.getPosition().converBsonTimeStamp()));
- } else {
- iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
+ if (!replicaSet.isRuning() || !replicaSetsContext.isInitSyncAbort()) {
+ return;
}
+
+ MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
+ FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find(
+ Filters.gt("ts", replicaSetConfig.getPosition().converBsonTimeStamp()));
+
MongoCursor<Document> cursor = iterable.sort(new Document("$natural", 1))
.noCursorTimeout(true)
.cursorType(CursorType.TailableAwait)
@@ -70,10 +96,26 @@ public class ReplicatorTask implements Runnable {
logger.info("replicaSet:{}, already shutdown, replicaTask end of life cycle", replicaSetConfig);
}
+ private BsonTimestamp findOplogFirstPosition() {
+ MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
+ FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
+ Document lastOplog = iterable.sort(new Document("$natural", 1)).limit(1).first();
+ BsonTimestamp timestamp = lastOplog.get(Constants.TIMESTAMP, BsonTimestamp.class);
+ return timestamp;
+ }
+
+ private void recordOplogLastPosition() {
+ MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
+ FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
+ Document lastOplog = iterable.sort(new Document("$natural", -1)).limit(1).first();
+ BsonTimestamp timestamp = lastOplog.get(Constants.TIMESTAMP, BsonTimestamp.class);
+ replicaSetConfig.setPosition(new Position(timestamp.getTime(), timestamp.getInc(), false));
+ }
+
private void executorCursor(MongoCursor<Document> cursor) {
while (cursor.hasNext() && !replicaSet.isPause()) {
Document document = cursor.next();
- ReplicationEvent event = EventConverter.convert(document, replicaSetConfig.getReplicaSetName());
+ ReplicationEvent event = Document2EventConverter.convert(document, replicaSetConfig.getReplicaSetName());
if (replicaSetsContext.filterEvent(event)) {
replicaSetsContext.publishEvent(event, replicaSetConfig);
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java b/src/main/java/org/apache/connect/mongo/replicator/event/Document2EventConverter.java
similarity index 58%
rename from src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java
rename to src/main/java/org/apache/connect/mongo/replicator/event/Document2EventConverter.java
index 1b48990..99ab707 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/Document2EventConverter.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator.event;
import java.util.Optional;
@@ -6,24 +23,24 @@ import org.bson.Document;
import static org.apache.connect.mongo.replicator.Constants.HASH;
import static org.apache.connect.mongo.replicator.Constants.NAMESPACE;
-import static org.apache.connect.mongo.replicator.Constants.OBJECTID;
+import static org.apache.connect.mongo.replicator.Constants.OBJECT_ID;
import static org.apache.connect.mongo.replicator.Constants.OPERATION;
-import static org.apache.connect.mongo.replicator.Constants.OPERATIONTYPE;
+import static org.apache.connect.mongo.replicator.Constants.OPERATION_TYPE;
import static org.apache.connect.mongo.replicator.Constants.TIMESTAMP;
import static org.apache.connect.mongo.replicator.Constants.VERSION;
-public class EventConverter {
+public class Document2EventConverter {
public static ReplicationEvent convert(Document document, String replicaSetName) {
ReplicationEvent event = new ReplicationEvent();
- event.setOperationType(OperationType.getOperationType(document.getString(OPERATIONTYPE)));
+ event.setOperationType(OperationType.getOperationType(document.getString(OPERATION_TYPE)));
event.setTimestamp(document.get(TIMESTAMP, BsonTimestamp.class));
event.setH(document.getLong(HASH));
event.setV(document.getInteger(VERSION));
event.setNamespace(document.getString(NAMESPACE));
event.setEventData(Optional.ofNullable(document.get(OPERATION, Document.class)));
- event.setObjectId(Optional.ofNullable(document.get(OBJECTID, Document.class)));
+ event.setObjectId(Optional.ofNullable(document.get(OBJECT_ID, Document.class)));
event.setReplicaSetName(replicaSetName);
event.setDocument(document);
return event;
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java b/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
index 54df394..b418666 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
@@ -1,28 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator.event;
+import org.apache.commons.lang3.StringUtils;
+
public enum OperationType {
INSERT("i"),
UPDATE("u"),
DELETE("d"),
NOOP("n"),
- DBCOMMAND("c"),
+ DB_COMMAND("c"),
CREATED("created"),
UNKNOWN("unknown");
- private final String operationStr;
+ private final String operation;
- OperationType(String operationStr) {
- this.operationStr = operationStr;
+ OperationType(String operation) {
+ this.operation = operation;
}
- public static OperationType getOperationType(String operationStr) {
- for (OperationType operationType : OperationType.values()) {
- if (operationType.operationStr.equals(operationStr)) {
- return operationType;
- }
+ public static OperationType getOperationType(String operation) {
+
+ if (StringUtils.isEmpty(operation)) {
+ return UNKNOWN;
+ }
+
+ switch (operation) {
+ case "i":
+ return INSERT;
+ case "u":
+ return UPDATE;
+ case "d":
+ return DELETE;
+ case "n":
+ return NOOP;
+ case "c":
+ return DB_COMMAND;
+ case "created":
+ return CREATED;
+ default:
+ return UNKNOWN;
}
- return UNKNOWN;
}
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java b/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
index 6407781..7adca71 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.connect.mongo.replicator.event;
import io.openmessaging.connector.api.data.EntryType;
diff --git a/src/test/java/org/apache/connect/mongo/FilterTest.java b/src/test/java/org/apache/connect/mongo/FilterTest.java
index a804b8b..31c8f4d 100644
--- a/src/test/java/org/apache/connect/mongo/FilterTest.java
+++ b/src/test/java/org/apache/connect/mongo/FilterTest.java
@@ -59,8 +59,9 @@ public class FilterTest {
ReplicationEvent replicationEvent = new ReplicationEvent();
replicationEvent.setOperationType(OperationType.NOOP);
Assert.assertFalse(filter.filterEvent(replicationEvent));
- replicationEvent.setOperationType(OperationType.DBCOMMAND);
+ replicationEvent.setOperationType(OperationType.DB_COMMAND);
Assert.assertTrue(filter.filterEvent(replicationEvent));
+
}
}
diff --git a/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java b/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
index 93adeeb..0f02064 100644
--- a/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
@@ -37,7 +37,7 @@ public class MongoFactoryTest {
@Test
public void testCreateMongoClientWithSSL() {
- sourceTaskConfig.setSsl("ssl");
+ sourceTaskConfig.setSsl(true);
MongoClientSettings settings = getSettings();
System.out.println(settings.getSslSettings());
Assert.assertTrue(settings.getSslSettings().isEnabled());
@@ -45,7 +45,7 @@ public class MongoFactoryTest {
@Test
public void testCreateMongoClientWithTSL() {
- sourceTaskConfig.setTsl("tsl");
+ sourceTaskConfig.setTsl(true);
MongoClientSettings settings = getSettings();
System.out.println(settings.getSslSettings());
Assert.assertTrue(settings.getSslSettings().isEnabled());
@@ -55,7 +55,7 @@ public class MongoFactoryTest {
public void testCreateMongoClientWithserverSelectionTimeoutMS() {
try {
replicaSetConfig.setReplicaSetName("testReplicatSet");
- sourceTaskConfig.setServerSelectionTimeoutMS("150");
+ sourceTaskConfig.setServerSelectionTimeoutMS(150);
System.out.println(getSettings().getClusterSettings());
Assert.assertTrue(getSettings().getClusterSettings().getServerSelectionTimeout(TimeUnit.MILLISECONDS) == 150);
} catch (MongoTimeoutException exception) {
@@ -65,7 +65,7 @@ public class MongoFactoryTest {
@Test
public void testCreateMongoClientWithConnectTimeoutMS() {
- sourceTaskConfig.setConnectTimeoutMS("1200");
+ sourceTaskConfig.setConnectTimeoutMS(1200);
System.out.println(getSettings().getSocketSettings());
Assert.assertTrue(getSettings().getSocketSettings().getConnectTimeout(TimeUnit.MILLISECONDS) == 1200);
@@ -73,40 +73,40 @@ public class MongoFactoryTest {
@Test
public void testCreateMongoClientWithSocketTimeoutMS() {
- sourceTaskConfig.setSocketTimeoutMS("1100");
+ sourceTaskConfig.setSocketTimeoutMS(1100);
System.out.println(getSettings().getSocketSettings());
Assert.assertTrue(getSettings().getSocketSettings().getReadTimeout(TimeUnit.MILLISECONDS) == 1100);
}
@Test
public void testCreateMongoClientWithInvalidHostNameAllowed() {
- sourceTaskConfig.setSslInvalidHostNameAllowed("true");
+ sourceTaskConfig.setSslInvalidHostNameAllowed(true);
System.out.println(getSettings().getSslSettings());
Assert.assertTrue(getSettings().getSslSettings().isInvalidHostNameAllowed());
- sourceTaskConfig.setSslInvalidHostNameAllowed("false");
+ sourceTaskConfig.setSslInvalidHostNameAllowed(false);
System.out.println(getSettings().getSslSettings());
Assert.assertFalse(getSettings().getSslSettings().isInvalidHostNameAllowed());
}
@Test
public void testCreateMongoClientWithInvalidHostNameAllowedTsl() {
- sourceTaskConfig.setTlsAllowInvalidHostnames("true");
+ sourceTaskConfig.setTlsAllowInvalidHostnames(true);
System.out.println(getSettings().getSslSettings());
Assert.assertTrue(getSettings().getSslSettings().isInvalidHostNameAllowed());
- sourceTaskConfig.setTlsAllowInvalidHostnames("false");
+ sourceTaskConfig.setTlsAllowInvalidHostnames(false);
System.out.println(getSettings().getSslSettings());
Assert.assertFalse(getSettings().getSslSettings().isInvalidHostNameAllowed());
}
@Test
public void testCreateMongoClientWithTlsinsecure() {
- sourceTaskConfig.setTlsInsecure("true");
+ sourceTaskConfig.setTlsInsecure(true);
System.out.println(getSettings().getSslSettings());
Assert.assertTrue(getSettings().getSslSettings().isInvalidHostNameAllowed());
- sourceTaskConfig.setTlsInsecure("false");
+ sourceTaskConfig.setTlsInsecure(false);
System.out.println(getSettings().getSslSettings());
Assert.assertFalse(getSettings().getSslSettings().isInvalidHostNameAllowed());
}
diff --git a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
index f85e4a9..cc02fbc 100644
--- a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
@@ -12,6 +12,7 @@ import java.util.Optional;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.connect.mongo.connector.MongoSourceConnector;
import org.apache.connect.mongo.connector.MongoSourceTask;
+import org.apache.connect.mongo.replicator.Position;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
import org.apache.connect.mongo.replicator.ReplicaSetsContext;
import org.apache.connect.mongo.replicator.event.OperationType;
@@ -73,7 +74,7 @@ public class MongoSourceConnectorTest {
Assert.assertEquals("testReplicaName", new String(sourcePartition.array()));
ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
- ReplicaSetConfig.Position position = JSONObject.parseObject(new String(sourcePosition.array()), ReplicaSetConfig.Position.class);
+ Position position = JSONObject.parseObject(new String(sourcePosition.array()), Position.class);
Assert.assertEquals(position.getTimeStamp(), 1565609506);
Assert.assertEquals(position.getInc(), 1);
Assert.assertEquals(position.isInitSync(), false);
diff --git a/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java b/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
index b696393..4983a66 100644
--- a/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
@@ -14,7 +14,6 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.apache.connect.mongo.connector.MongoSourceTask;
-import org.apache.connect.mongo.replicator.Constants;
import org.apache.connect.mongo.replicator.ReplicaSet;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
import org.apache.connect.mongo.replicator.ReplicaSetsContext;
@@ -31,7 +30,7 @@ public class MongoSourceTaskTest {
defaultKeyValue.put("positionTimeStamp", "11111111");
defaultKeyValue.put("positionInc", "111");
defaultKeyValue.put("serverSelectionTimeoutMS", "10");
- defaultKeyValue.put("dataSync", Constants.INITSYNC);
+ defaultKeyValue.put("dataSync", "true");
Field context = SourceTask.class.getDeclaredField("context");
context.setAccessible(true);
diff --git a/src/test/java/org/apache/connect/mongo/MongoTest.java b/src/test/java/org/apache/connect/mongo/MongoTest.java
index 98e9a42..3d900fa 100644
--- a/src/test/java/org/apache/connect/mongo/MongoTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoTest.java
@@ -19,10 +19,11 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.connect.mongo.initsync.InitSync;
import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.replicator.Position;
import org.apache.connect.mongo.replicator.ReplicaSet;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
import org.apache.connect.mongo.replicator.ReplicaSetsContext;
-import org.apache.connect.mongo.replicator.event.EventConverter;
+import org.apache.connect.mongo.replicator.event.Document2EventConverter;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
import org.bson.BsonTimestamp;
@@ -49,11 +50,11 @@ public class MongoTest {
oplog.put(Constants.TIMESTAMP, timestamp);
oplog.put(Constants.NAMESPACE, "test.person");
oplog.put(Constants.HASH, 11111L);
- oplog.put(Constants.OPERATIONTYPE, "i");
+ oplog.put(Constants.OPERATION_TYPE, "i");
Document document = new Document();
document.put("test", "test");
oplog.put(Constants.OPERATION, document);
- ReplicationEvent event = EventConverter.convert(oplog, "testR");
+ ReplicationEvent event = Document2EventConverter.convert(oplog, "testR");
Assert.assertEquals(timestamp, event.getTimestamp());
Assert.assertEquals("test.person", event.getNamespace());
Assert.assertTrue(11111L == event.getH());
@@ -95,16 +96,16 @@ public class MongoTest {
int syncCount = 0;
while (syncCount < count) {
Collection<SourceDataEntry> sourceDataEntries = replicaSetsContext.poll();
- Assert.assertNotNull(sourceDataEntries);
+ Assert.assertTrue(sourceDataEntries.size() > 0);
for (SourceDataEntry sourceDataEntry : sourceDataEntries) {
ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
Assert.assertEquals("test", new String(sourcePartition.array()));
ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
- ReplicaSetConfig.Position position = replicaSetConfig.emptyPosition();
+ Position position = new Position();
position.setInitSync(true);
position.setTimeStamp(0);
position.setInc(0);
- Assert.assertEquals(position, JSONObject.parseObject(new String(sourcePosition.array()), ReplicaSetConfig.Position.class));
+ Assert.assertEquals(position, JSONObject.parseObject(new String(sourcePosition.array()), Position.class));
EntryType entryType = sourceDataEntry.getEntryType();
Assert.assertEquals(EntryType.CREATE, entryType);
String queueName = sourceDataEntry.getQueueName();
@@ -122,4 +123,16 @@ public class MongoTest {
Assert.assertTrue(syncCount == count);
}
+
+ @Test
+ public void testCompareBsonTimestamp() {
+ BsonTimestamp lt = new BsonTimestamp(11111111, 1);
+ BsonTimestamp gt = new BsonTimestamp(11111111, 2);
+ Assert.assertTrue(lt.compareTo(gt) < 0);
+
+ lt = new BsonTimestamp(11111111, 1);
+ gt = new BsonTimestamp(22222222, 1);
+ Assert.assertTrue(lt.compareTo(gt) < 0);
+
+ }
}
diff --git a/src/test/java/org/apache/connect/mongo/OperationTypeTest.java b/src/test/java/org/apache/connect/mongo/OperationTypeTest.java
new file mode 100644
index 0000000..d8c5a9b
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/OperationTypeTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.connect.mongo;
+
+import org.apache.connect.mongo.replicator.event.OperationType;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class OperationTypeTest {
+
+ @Test
+ public void testGetOperationType() {
+ Assert.assertEquals(OperationType.INSERT, OperationType.getOperationType("i"));
+ Assert.assertEquals(OperationType.UPDATE, OperationType.getOperationType("u"));
+ Assert.assertEquals(OperationType.DELETE, OperationType.getOperationType("d"));
+ Assert.assertEquals(OperationType.NOOP, OperationType.getOperationType("n"));
+ Assert.assertEquals(OperationType.DB_COMMAND, OperationType.getOperationType("c"));
+ Assert.assertEquals(OperationType.CREATED, OperationType.getOperationType("created"));
+ Assert.assertEquals(OperationType.UNKNOWN, OperationType.getOperationType("test"));
+
+ }
+}
diff --git a/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java b/src/test/java/org/apache/connect/mongo/ReplicaSetManagerTest.java
similarity index 75%
rename from src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java
rename to src/test/java/org/apache/connect/mongo/ReplicaSetManagerTest.java
index 5276f4f..1d3b743 100644
--- a/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java
+++ b/src/test/java/org/apache/connect/mongo/ReplicaSetManagerTest.java
@@ -2,26 +2,26 @@ package org.apache.connect.mongo;
import java.util.Map;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
-import org.apache.connect.mongo.replicator.ReplicaSets;
+import org.apache.connect.mongo.replicator.ReplicaSetManager;
import org.junit.Assert;
import org.junit.Test;
-public class ReplicaSetsTest {
+public class ReplicaSetManagerTest {
@Test(expected = IllegalArgumentException.class)
public void testCreatReplicaSetsExceptionWithoutMongoAddr() {
- ReplicaSets.create("");
+ ReplicaSetManager.create("");
}
@Test(expected = IllegalArgumentException.class)
public void testCreatReplicaSetsExceptioWithoutReplicaSetName() {
- ReplicaSets.create("127.0.0.1:27081");
+ ReplicaSetManager.create("127.0.0.1:27081");
}
@Test
public void testCreatReplicaSetsSpecialReplicaSetName() {
- ReplicaSets replicaSets = ReplicaSets.create("replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083");
- Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
+ ReplicaSetManager replicaSetManager = ReplicaSetManager.create("replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083");
+ Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSetManager.getReplicaConfigByName();
Assert.assertTrue(replicaSetConfigMap.size() == 1);
Assert.assertNotNull(replicaSetConfigMap.get("replicaName1"));
Assert.assertEquals("127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083", replicaSetConfigMap.get("replicaName1").getHost());
@@ -30,8 +30,8 @@ public class ReplicaSetsTest {
@Test
public void testCreatReplicaSetsSpecialShardNameAndReplicaSetName() {
- ReplicaSets replicaSets = ReplicaSets.create("shardName1=replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083");
- Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
+ ReplicaSetManager replicaSetManager = ReplicaSetManager.create("shardName1=replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083");
+ Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSetManager.getReplicaConfigByName();
Assert.assertTrue(replicaSetConfigMap.size() == 1);
Assert.assertNotNull(replicaSetConfigMap.get("replicaName1"));
Assert.assertEquals("127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083", replicaSetConfigMap.get("replicaName1").getHost());
@@ -41,8 +41,8 @@ public class ReplicaSetsTest {
@Test
public void testCreatReplicaSetsMutiMongoAddr() {
- ReplicaSets replicaSets = ReplicaSets.create("shardName1=replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083;shardName2=replicaName2/127.0.0.1:27281,127.0.0.1:27282,127.0.0.1:27283");
- Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
+ ReplicaSetManager replicaSetManager = ReplicaSetManager.create("shardName1=replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083;shardName2=replicaName2/127.0.0.1:27281,127.0.0.1:27282,127.0.0.1:27283");
+ Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSetManager.getReplicaConfigByName();
Assert.assertTrue(replicaSetConfigMap.size() == 2);
Assert.assertNotNull(replicaSetConfigMap.get("replicaName1"));
Assert.assertEquals("127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083", replicaSetConfigMap.get("replicaName1").getHost());
[rocketmq-connect] 03/13: 修改stop
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit e84812e504d44a3224eacb689646e4dc3494e280
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Tue Aug 6 14:36:04 2019 +0800
修改stop
---
src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java | 3 ---
src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java | 3 ---
2 files changed, 6 deletions(-)
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
index e116cfb..40dfdd0 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
@@ -88,11 +88,8 @@ public class MongoSourceTask extends SourceTask {
mongoReplicator.start();
}catch (Throwable throwable) {
logger.info("task start error", throwable);
- }finally {
stop();
}
-
-
}
@Override
diff --git a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
index 326ba03..a14ceee 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
@@ -56,17 +56,14 @@ public class MongoReplicator {
this.clientSettings = MongoClientSettings.builder().applicationName(APPLICATION_NAME)
.applyConnectionString(connectionString)
- .retryWrites(true)
.build();
this.mongoClient = MongoClients.create(clientSettings);
this.isReplicaMongo();
executorService.submit(new ReplicatorTask(this, mongoClient, mongoReplicatorConfig, filter));
}catch (Exception e) {
logger.info("start replicator error", e);
- }finally {
shutdown();
}
-
}
[rocketmq-connect] 04/13: fix some bug
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 32d316512cfb0e31d9255235695cbead65799891
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Thu Aug 8 16:18:41 2019 +0800
fix some bug
---
.../connect/mongo/MongoReplicatorConfig.java | 45 +++++----
.../connect/mongo/connector/MongoSourceTask.java | 14 ++-
.../apache/connect/mongo/initsync/InitSync.java | 38 +++-----
.../apache/connect/mongo/replicator/Filter.java | 80 ++++++++--------
.../connect/mongo/replicator/MongoReplicator.java | 23 +++--
.../replicator/event/DocumentConvertEvent.java | 24 ++---
.../java/org/apache/connect/mongo/FilterTest.java | 42 +++++----
.../java/org/apache/connect/mongo/MongoTest.java | 105 +++++++++++++++++++++
.../org/apache/connect/mongo/ReplicatorTest.java | 33 -------
9 files changed, 228 insertions(+), 176 deletions(-)
diff --git a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java b/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
index 18a834f..9f17aab 100644
--- a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
+++ b/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
@@ -1,6 +1,7 @@
package org.apache.connect.mongo;
import io.openmessaging.KeyValue;
+import org.apache.commons.lang3.StringUtils;
import java.lang.reflect.Method;
import java.util.HashSet;
@@ -8,12 +9,11 @@ import java.util.Set;
public class MongoReplicatorConfig {
+ private String replicaSet;
private String mongoAddr;
- private int mongoPort;
private String mongoUserName;
private String mongoPassWord;
- private String interestDB;
- private String interestCollection;
+ private String interestDbAndCollection;
private long positionTimeStamp;
private int positionInc;
private String dataSync;
@@ -23,7 +23,6 @@ public class MongoReplicatorConfig {
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
{
add("mongoAddr");
- add("mongoPort");
}
};
@@ -51,20 +50,12 @@ public class MongoReplicatorConfig {
this.positionTimeStamp = positionTimeStamp;
}
- public String getInterestDB() {
- return interestDB;
+ public String getInterestDbAndCollection() {
+ return interestDbAndCollection;
}
- public void setInterestDB(String interestDB) {
- this.interestDB = interestDB;
- }
-
- public String getInterestCollection() {
- return interestCollection;
- }
-
- public void setInterestCollection(String interestCollection) {
- this.interestCollection = interestCollection;
+ public void setInterestDbAndCollection(String interestDbAndCollection) {
+ this.interestDbAndCollection = interestDbAndCollection;
}
public String getMongoAddr() {
@@ -75,13 +66,6 @@ public class MongoReplicatorConfig {
this.mongoAddr = mongoAddr;
}
- public int getMongoPort() {
- return mongoPort;
- }
-
- public void setMongoPort(int mongoPort) {
- this.mongoPort = mongoPort;
- }
public String getMongoUserName() {
return mongoUserName;
@@ -109,6 +93,14 @@ public class MongoReplicatorConfig {
}
+ public String getReplicaSet() {
+ return replicaSet;
+ }
+
+ public void setReplicaSet(String replicaSet) {
+ this.replicaSet = replicaSet;
+ }
+
public void load(KeyValue props) {
properties2Object(props, this);
@@ -154,4 +146,11 @@ public class MongoReplicatorConfig {
}
}
}
+
+ public String getDataSouce() {
+ if (StringUtils.isBlank(replicaSet)) {
+ return mongoAddr;
+ }
+ return replicaSet + ":" + mongoAddr;
+ }
}
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
index 40dfdd0..9176ab7 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
@@ -44,7 +44,7 @@ public class MongoSourceTask extends SourceTask {
buildFieleds(schema);
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(event.getNamespace())
+ .queue(event.getNamespace().replace(".", "-"))
.entryType(event.getEntryType());
if (event.getOperationType().ordinal() == OperationType.CREATED.ordinal()) {
@@ -71,9 +71,7 @@ public class MongoSourceTask extends SourceTask {
replicatorConfig = new MongoReplicatorConfig();
replicatorConfig.load(config);
mongoReplicator = new MongoReplicator(replicatorConfig);
- mongoSource = new StringBuilder()
- .append(replicatorConfig.getMongoAddr())
- .append(replicatorConfig.getMongoPort()).toString();
+ mongoSource = replicatorConfig.getDataSouce();
ByteBuffer position = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(
mongoSource.getBytes()));
@@ -121,7 +119,7 @@ public class MongoSourceTask extends SourceTask {
schema.getFields().add(operation);
Field patch = new Field(5, Constants.PATCH, FieldType.STRING);
schema.getFields().add(patch);
- Field objectId = new Field(5, Constants.OBJECTID, FieldType.STRING);
+ Field objectId = new Field(6, Constants.OBJECTID, FieldType.STRING);
schema.getFields().add(objectId);
}
@@ -134,9 +132,9 @@ public class MongoSourceTask extends SourceTask {
jsonObject.put(Constants.INITSYNC, true);
break;
default:
- jsonObject.put(Constants.POSITION_TIMESTAMP, 0);
- jsonObject.put(Constants.POSITION_INC, 0);
- jsonObject.put(Constants.INITSYNC, true);
+ jsonObject.put(Constants.POSITION_TIMESTAMP, event.getTimestamp().getTime());
+ jsonObject.put(Constants.POSITION_INC, event.getTimestamp().getInc());
+ jsonObject.put(Constants.INITSYNC, false);
break;
}
return jsonObject;
diff --git a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
index 4b3a238..e12412b 100644
--- a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
+++ b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
@@ -27,7 +27,6 @@ public class InitSync {
private MongoClient mongoClient;
private Filter filter;
private int copyThreadCount;
- private Set<String> interestDataBases;
private Set<CollectionMeta> interestCollections;
private CountDownLatch countDownLatch;
private MongoReplicator mongoReplicator;
@@ -53,8 +52,7 @@ public class InitSync {
}
private void init() {
- interestDataBases = getInterestDataBase();
- interestCollections = getInterestCollection(interestDataBases);
+ interestCollections = getInterestCollection();
copyThreadCount = Math.min(interestCollections.size(), mongoReplicatorConfig.getCopyThread());
copyExecutor = Executors.newFixedThreadPool(copyThreadCount, new ThreadFactory() {
@@ -68,15 +66,17 @@ public class InitSync {
countDownLatch = new CountDownLatch(interestCollections.size());
}
- private Set<CollectionMeta> getInterestCollection(Set<String> interestDataBases) {
+ private Set<CollectionMeta> getInterestCollection() {
Set<CollectionMeta> res = new HashSet<>();
- for (String interestDataBase : interestDataBases) {
- MongoIterable<String> collectionNames = mongoClient.getDatabase(interestDataBase).listCollectionNames();
- MongoCursor<String> iterator = collectionNames.iterator();
- while (iterator.hasNext()) {
- String collectionName = iterator.next();
- if (filter.filterCollectionName(collectionName)) {
- CollectionMeta collectionMeta = new CollectionMeta(interestDataBase, collectionName);
+ MongoIterable<String> databaseNames = mongoClient.listDatabaseNames();
+ MongoCursor<String> dbIterator = databaseNames.iterator();
+ while (dbIterator.hasNext()) {
+ String dataBaseName = dbIterator.next();
+ MongoCursor<String> collIterator = mongoClient.getDatabase(dataBaseName).listCollectionNames().iterator();
+ while (collIterator.hasNext()) {
+ String collectionName = collIterator.next();
+ CollectionMeta collectionMeta = new CollectionMeta(dataBaseName, collectionName);
+ if (filter.filter(collectionMeta)) {
res.add(collectionMeta);
}
}
@@ -86,19 +86,6 @@ public class InitSync {
}
- private Set<String> getInterestDataBase() {
- Set<String> res = new HashSet<>();
- MongoIterable<String> databaseNames = mongoClient.listDatabaseNames();
- MongoCursor<String> iterator = databaseNames.iterator();
- while (iterator.hasNext()) {
- String dataBaseName = iterator.next();
- if (filter.filterDatabaseName(dataBaseName)) {
- res.add(dataBaseName);
- }
- }
-
- return res;
- }
class CopyRunner implements Runnable {
@@ -132,6 +119,9 @@ public class InitSync {
event.setNamespace(collectionMeta.getNameSpace());
mongoReplicator.publishEvent(event);
}
+
+ } catch (Exception e) {
+ logger.error("init sync database:{}, collection:{} error", collectionMeta.getDatabaseName(), collectionMeta.getNameSpace());
} finally {
countDownLatch.countDown();
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Filter.java b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
index 05dec54..4a62e41 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/Filter.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
@@ -1,74 +1,68 @@
package org.apache.connect.mongo.replicator;
-import org.apache.commons.lang3.ArrayUtils;
+import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.TypeReference;
import org.apache.commons.lang3.StringUtils;
import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.initsync.CollectionMeta;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.function.Function;
public class Filter {
- private Function<String, Boolean> dbFilter;
- private Function<String, Boolean> collectionFilter;
- private Function<OperationType, Boolean> noopFilter;
+ private Function<CollectionMeta, Boolean> dbAndCollectionFilter;
+ private Map<String, List<String>> interestMap = new HashMap<>();
+ private Function<OperationType, Boolean> notNoopFilter;
public Filter(MongoReplicatorConfig mongoReplicatorConfig) {
- if (StringUtils.isNotBlank(mongoReplicatorConfig.getInterestDB())) {
- dbFilter = (dataBaseName) -> {
- if (StringUtils.isBlank(dataBaseName)) {
- return true;
- }
- String interestDB = mongoReplicatorConfig.getInterestDB();
- String[] db = StringUtils.split(interestDB, ",");
- if (ArrayUtils.contains(db, dataBaseName)) {
- return true;
- }
- return false;
- };
- } else {
- dbFilter = (dataBaseName) -> true;
- }
+ String interestDbAndCollection = mongoReplicatorConfig.getInterestDbAndCollection();
+ if (StringUtils.isNotBlank(interestDbAndCollection)) {
+ JSONObject jsonObject = JSONObject.parseObject(interestDbAndCollection);
+ for (String db : jsonObject.keySet()) {
+ List<String> collections = jsonObject.getObject(db, new TypeReference<List<String>>() {
+ });
+ interestMap.put(db, collections);
+ }
- if (StringUtils.isNotBlank(mongoReplicatorConfig.getInterestCollection())) {
- collectionFilter = (collectionName) -> {
- if (StringUtils.isBlank(collectionName)) {
- return true;
- }
- String interestCollection = mongoReplicatorConfig.getInterestCollection();
- String[] coll = StringUtils.split(interestCollection, ",");
- if (ArrayUtils.contains(coll, collectionName)) {
- return true;
- }
- return false;
- };
- } else {
- collectionFilter = (collectionName) -> true;
}
- noopFilter = (opeartionType) -> opeartionType.ordinal() != OperationType.NOOP.ordinal();
- }
+ dbAndCollectionFilter = (collectionMeta) -> {
+ if (interestMap.size() == 0) {
+ return true;
+ }
+ List<String> collections = interestMap.get(collectionMeta.getDatabaseName());
+ if (collections == null || collections.size() == 0) {
+ return false;
+ }
+
+ if (collections.contains("*") || collections.contains(collectionMeta.getCollectionName())) {
+ return true;
+ }
- public boolean filterDatabaseName(String dataBaseName) {
- return dbFilter.apply(dataBaseName);
+ return false;
+ };
+
+ notNoopFilter = (opeartionType) -> opeartionType.ordinal() != OperationType.NOOP.ordinal();
}
- public boolean filterCollectionName(String collectionName) {
- return collectionFilter.apply(collectionName);
+ public boolean filter(CollectionMeta collectionMeta) {
+ return dbAndCollectionFilter.apply(collectionMeta);
}
public boolean filterEvent(ReplicationEvent event) {
- return dbFilter.apply(event.getDatabaseName())
- && collectionFilter.apply(event.getCollectionName())
- && noopFilter.apply(event.getOperationType());
+ return dbAndCollectionFilter.apply(new CollectionMeta(event.getDatabaseName(), event.getCollectionName()))
+ && notNoopFilter.apply(event.getOperationType());
}
-
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
index a14ceee..0f8520b 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
@@ -54,7 +54,8 @@ public class MongoReplicator {
return;
}
- this.clientSettings = MongoClientSettings.builder().applicationName(APPLICATION_NAME)
+ this.clientSettings = MongoClientSettings.builder()
+ .applicationName(APPLICATION_NAME)
.applyConnectionString(connectionString)
.build();
this.mongoClient = MongoClients.create(clientSettings);
@@ -68,7 +69,6 @@ public class MongoReplicator {
private void buildConnectionString() {
- checkConfig();
StringBuilder sb = new StringBuilder();
sb.append("mongodb://");
if (StringUtils.isNotBlank(mongoReplicatorConfig.getMongoUserName())
@@ -80,19 +80,17 @@ public class MongoReplicator {
}
sb.append(mongoReplicatorConfig.getMongoAddr());
- sb.append(":");
- sb.append(mongoReplicatorConfig.getMongoPort());
-
+ sb.append("/");
+ if (StringUtils.isBlank(mongoReplicatorConfig.getReplicaSet())) {
+ sb.append("?");
+ sb.append("replicaSet=");
+ sb.append(mongoReplicatorConfig.getReplicaSet());
+ }
this.connectionString = new ConnectionString(sb.toString());
}
- private void checkConfig() {
- Validate.notBlank(mongoReplicatorConfig.getMongoAddr(), "mongo url is blank");
- Validate.isTrue(mongoReplicatorConfig.getMongoPort() > 0 && mongoReplicatorConfig.getMongoPort() < 65535, "mongo port should >0 and <65535");
- }
-
- private boolean isReplicaMongo() {
+ public boolean isReplicaMongo() {
MongoDatabase local = mongoClient.getDatabase(MONGO_LOCAL_DATABASE);
MongoIterable<String> collectionNames = local.listCollectionNames();
for (String collectionName : collectionNames) {
@@ -101,7 +99,7 @@ public class MongoReplicator {
}
}
this.shutdown();
- throw new IllegalStateException(String.format("url:%s, port:%s is not replica", mongoReplicatorConfig.getMongoAddr(), mongoReplicatorConfig.getMongoPort()));
+ throw new IllegalStateException(String.format("url:%s, set:%s is not replica", mongoReplicatorConfig.getMongoAddr(), mongoReplicatorConfig.getReplicaSet()));
}
public void shutdown() {
@@ -127,6 +125,7 @@ public class MongoReplicator {
}
+
public void pause() {
pause = true;
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java b/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java
index 6b902e7..a18aa52 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java
@@ -10,26 +10,20 @@ import static org.apache.connect.mongo.replicator.Constants.*;
public class DocumentConvertEvent {
+
public static ReplicationEvent convert(Document document) {
- ReplicationEvent event = null;
- try {
- OperationType operationType = OperationType.getOperationType(document.getString(OPERATIONTYPE));
- BsonTimestamp timestamp = (BsonTimestamp) document.get(TIMESTAMP);
+ OperationType operationType = OperationType.getOperationType(document.getString(OPERATIONTYPE));
+ BsonTimestamp timestamp = (BsonTimestamp) document.get(TIMESTAMP);
// Long t = document.getLong("t");
- Long h = document.getLong(HASH);
- Integer v = document.getInteger(VERSION);
- String nameSpace = document.getString(NAMESPACE);
+ Long h = document.getLong(HASH);
+ Integer v = document.getInteger(VERSION);
+ String nameSpace = document.getString(NAMESPACE);
// String uuid = document.getString("uuid");
// Date wall = document.getDate("wall");
- Document operation = document.get(OPERATION, Document.class);
- Document objectID = document.get(OBJECTID, Document.class);
- event = new ReplicationEvent(operationType, timestamp, v, h, nameSpace, Optional.ofNullable(operation), Optional.ofNullable(objectID), document);
- } catch (Exception e) {
- System.out.println(e);
- }
-
- return event;
+ Document operation = document.get(OPERATION, Document.class);
+ Document objectID = document.get(OBJECTID, Document.class);
+ return new ReplicationEvent(operationType, timestamp, v, h, nameSpace, Optional.ofNullable(operation), Optional.ofNullable(objectID), document);
}
}
diff --git a/src/test/java/org/apache/connect/mongo/FilterTest.java b/src/test/java/org/apache/connect/mongo/FilterTest.java
index 24f36d9..2b14b13 100644
--- a/src/test/java/org/apache/connect/mongo/FilterTest.java
+++ b/src/test/java/org/apache/connect/mongo/FilterTest.java
@@ -1,5 +1,7 @@
package org.apache.connect.mongo;
+import com.alibaba.fastjson.JSONObject;
+import org.apache.connect.mongo.initsync.CollectionMeta;
import org.apache.connect.mongo.replicator.Filter;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
@@ -7,51 +9,55 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
public class FilterTest {
private MongoReplicatorConfig config;
-
+ private Map<String, List<String>> insterest;
@Before
public void init() {
config = new MongoReplicatorConfig();
-
+ insterest = new HashMap<>();
}
@Test
public void testSpecialDb() {
- config.setInterestDB("test,admin");
+ List<String> collections = new ArrayList<>();
+ collections.add("person");
+ insterest.put("test", collections);
+ config.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
Filter filter = new Filter(config);
- Assert.assertTrue(filter.filterDatabaseName("test"));
- Assert.assertFalse(filter.filterDatabaseName("test01"));
+ Assert.assertTrue(filter.filter(new CollectionMeta("test", "person")));
+ Assert.assertFalse(filter.filter(new CollectionMeta("test", "person01")));
}
@Test
public void testBlankDb() {
Filter filter = new Filter(config);
- Assert.assertTrue(filter.filterDatabaseName("test"));
- Assert.assertTrue(filter.filterDatabaseName("test01"));
+ Assert.assertTrue(filter.filter(new CollectionMeta("test" ,"test")));
+ Assert.assertTrue(filter.filter(new CollectionMeta("test1" ,"test01")));
}
@Test
- public void testSpecialCollection() {
- config.setInterestCollection("test,admin");
+ public void testAsterisk() {
+ List<String> collections = new ArrayList<>();
+ collections.add("*");
+ insterest.put("test", collections);
+ config.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
Filter filter = new Filter(config);
- Assert.assertTrue(filter.filterCollectionName("test"));
- Assert.assertFalse(filter.filterCollectionName("test01"));
+ Assert.assertTrue(filter.filter(new CollectionMeta("test", "testsad")));
+ Assert.assertTrue(filter.filter(new CollectionMeta("test", "tests032")));
}
- @Test
- public void testBlankCollection() {
- Filter filter = new Filter(config);
- Assert.assertTrue(filter.filterCollectionName("test"));
- Assert.assertTrue(filter.filterCollectionName("test01"));
- }
-
@Test
public void testFilterEvent() {
diff --git a/src/test/java/org/apache/connect/mongo/MongoTest.java b/src/test/java/org/apache/connect/mongo/MongoTest.java
new file mode 100644
index 0000000..849c00c
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/MongoTest.java
@@ -0,0 +1,105 @@
+package org.apache.connect.mongo;
+
+import com.alibaba.fastjson.JSONObject;
+import com.mongodb.ConnectionString;
+import com.mongodb.CursorType;
+import com.mongodb.MongoClientSettings;
+import com.mongodb.client.*;
+import com.mongodb.client.model.Filters;
+import io.openmessaging.connector.api.data.EntryType;
+import org.apache.connect.mongo.initsync.InitSync;
+import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.replicator.Filter;
+import org.apache.connect.mongo.replicator.MongoReplicator;
+import org.apache.connect.mongo.replicator.event.DocumentConvertEvent;
+import org.apache.connect.mongo.replicator.event.OperationType;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.bson.BsonTimestamp;
+import org.bson.Document;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class MongoTest {
+
+ private MongoClient mongoClient;
+
+ @Before
+ public void before() {
+ MongoClientSettings.Builder builder = MongoClientSettings.builder();
+ builder.applyConnectionString(new ConnectionString("mongodb://127.0.0.1:27077"));
+ mongoClient = MongoClients.create(builder.build());
+ }
+
+
+ @Test
+ public void testConvertEvent() {
+ Document oplog = new Document();
+ BsonTimestamp timestamp = new BsonTimestamp(1565074665, 10);
+ oplog.put(Constants.TIMESTAMP, timestamp);
+ oplog.put(Constants.NAMESPACE, "test.person");
+ oplog.put(Constants.HASH, 11111L);
+ oplog.put(Constants.OPERATIONTYPE, "i");
+ Document document = new Document();
+ document.put("test", "test");
+ oplog.put(Constants.OPERATION, document);
+ ReplicationEvent event = DocumentConvertEvent.convert(oplog);
+ Assert.assertEquals(timestamp, event.getTimestamp());
+ Assert.assertEquals("test.person", event.getNamespace());
+ Assert.assertTrue(11111L == event.getH());
+ Assert.assertEquals(OperationType.INSERT, event.getOperationType());
+ Assert.assertEquals(EntryType.CREATE, event.getEntryType());
+ Assert.assertEquals(document, event.getEventData().get());
+
+
+ }
+
+
+ @Test
+ public void testInitSyncCopy() throws NoSuchFieldException, IllegalAccessException, InterruptedException {
+ MongoCollection<Document> collection = mongoClient.getDatabase("test").getCollection("person");
+ collection.deleteMany(new Document());
+ int count = 100;
+ List<Document> documents = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ Document document = new Document();
+ document.put("name", "test" + i);
+ document.put("age", i);
+ document.put("sex", i % 2 == 0 ? "boy" : "girl");
+ collection.insertOne(document);
+ documents.add(document);
+ }
+ MongoReplicatorConfig config = new MongoReplicatorConfig();
+ Map<String, List<String>> insterest = new HashMap<>();
+ List<String> collections = new ArrayList<>();
+ collections.add("*");
+ insterest.put("test", collections);
+ config.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
+ MongoReplicator mongoReplicator = new MongoReplicator(config);
+ Field running = MongoReplicator.class.getDeclaredField("running");
+ running.setAccessible(true);
+ running.set(mongoReplicator, new AtomicBoolean(true));
+ InitSync initSync = new InitSync(config, mongoClient, new Filter(config), mongoReplicator);
+ initSync.start();
+ BlockingQueue<ReplicationEvent> queue = mongoReplicator.getQueue();
+ while (count > 0) {
+ count--;
+ ReplicationEvent event = queue.poll(100, TimeUnit.MILLISECONDS);
+ Assert.assertTrue(event.getOperationType().equals(OperationType.CREATED));
+ Assert.assertNotNull(event.getDocument());
+ Assert.assertTrue(documents.contains(event.getDocument()));
+ }
+
+
+
+ }
+}
diff --git a/src/test/java/org/apache/connect/mongo/ReplicatorTest.java b/src/test/java/org/apache/connect/mongo/ReplicatorTest.java
deleted file mode 100644
index b0319f1..0000000
--- a/src/test/java/org/apache/connect/mongo/ReplicatorTest.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.connect.mongo;
-
-import org.apache.connect.mongo.replicator.MongoReplicator;
-import org.junit.Before;
-import org.junit.Test;
-
-public class ReplicatorTest {
-
- private MongoReplicatorConfig config;
-
- @Before
- public void before() {
- config = new MongoReplicatorConfig();
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testNoPort() {
- config.setMongoAddr("127.0.0.1");
- MongoReplicator mongoReplicator = new MongoReplicator(config);
- mongoReplicator.start();
- }
-
-
- @Test
- public void testNoPort1() {
- config.setMongoAddr("127.0.0.1");
- config.setMongoPort(27012);
- MongoReplicator mongoReplicator = new MongoReplicator(config);
- mongoReplicator.start();
- }
-
-
-}
[rocketmq-connect] 01/13: add junit test and modify some code
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit f629088918a2cc725326b981749798c0ba4b64de
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Mon Aug 5 15:01:55 2019 +0800
add junit test and modify some code
---
pom.xml | 180 +++++++++++++++++++++
.../connect/mongo/MongoReplicatorConfig.java | 157 ++++++++++++++++++
.../mongo/connector/MongoSourceConnector.java | 61 +++++++
.../connect/mongo/connector/MongoSourceTask.java | 148 +++++++++++++++++
.../connect/mongo/initsync/CollectionMeta.java | 41 +++++
.../apache/connect/mongo/initsync/InitSync.java | 146 +++++++++++++++++
.../apache/connect/mongo/replicator/Filter.java | 74 +++++++++
.../connect/mongo/replicator/MongoReplicator.java | 152 +++++++++++++++++
.../connect/mongo/replicator/ReplicatorTask.java | 84 ++++++++++
.../java/org/apache/connect/mongo/FilterTest.java | 67 ++++++++
.../connect/mongo/MongoSourceConnectorTest.java | 45 ++++++
.../org/apache/connect/mongo/ReplicatorTest.java | 33 ++++
12 files changed, 1188 insertions(+)
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..7ea7ab1
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,180 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-connect-mongo</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+
+ <name>connect-mongo</name>
+ <url>https://github.com/apache/incubator-rocketmq-externals/tree/master/rocketmq-connect-mongo</url>
+
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ </license>
+ </licenses>
+
+ <issueManagement>
+ <system>jira</system>
+ <url>https://issues.apache.org/jira/browse/RocketMQ</url>
+ </issueManagement>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+
+ <!-- Compiler settings properties -->
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>versions-maven-plugin</artifactId>
+ <version>2.3</version>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>clirr-maven-plugin</artifactId>
+ <version>2.7</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.6.1</version>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerVersion>${maven.compiler.source}</compilerVersion>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.19.1</version>
+ <configuration>
+ <argLine>-Xms512m -Xmx1024m</argLine>
+ <forkMode>always</forkMode>
+ <includes>
+ <include>**/*Test.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.6</version>
+ <configuration>
+ <locales>en_US</locales>
+ <outputEncoding>UTF-8</outputEncoding>
+ <inputEncoding>UTF-8</inputEncoding>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>3.0.1</version>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.10.4</version>
+ <configuration>
+ <charset>UTF-8</charset>
+ <locale>en_US</locale>
+ <excludePackageNames>io.openmessaging.internal</excludePackageNames>
+ </configuration>
+ <executions>
+ <execution>
+ <id>aggregate</id>
+ <goals>
+ <goal>aggregate</goal>
+ </goals>
+ <phase>site</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>3.0.2</version>
+ <configuration>
+ <encoding>${project.build.sourceEncoding}</encoding>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <version>3.0.4</version>
+ </plugin>
+ </plugins>
+ </build>
+
+
+ <dependencies>
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongodb-driver</artifactId>
+ <version>3.10.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.9</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.26</version>
+ </dependency>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-connector</artifactId>
+ <version>0.1.0-beta</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.0.13</version>
+ </dependency>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-api</artifactId>
+ <version>1.0.0-alpha</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-openmessaging</artifactId>
+ <version>4.3.2</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java b/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
new file mode 100644
index 0000000..18a834f
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
@@ -0,0 +1,157 @@
+package org.apache.connect.mongo;
+
+import io.openmessaging.KeyValue;
+
+import java.lang.reflect.Method;
+import java.util.HashSet;
+import java.util.Set;
+
+public class MongoReplicatorConfig {
+
+ private String mongoAddr;
+ private int mongoPort;
+ private String mongoUserName;
+ private String mongoPassWord;
+ private String interestDB;
+ private String interestCollection;
+ private long positionTimeStamp;
+ private int positionInc;
+ private String dataSync;
+ private int copyThread = Runtime.getRuntime().availableProcessors();
+
+
+ public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
+ {
+ add("mongoAddr");
+ add("mongoPort");
+ }
+ };
+
+ public int getPositionInc() {
+ return positionInc;
+ }
+
+ public void setPositionInc(int positionInc) {
+ this.positionInc = positionInc;
+ }
+
+ public int getCopyThread() {
+ return copyThread;
+ }
+
+ public void setCopyThread(int copyThread) {
+ this.copyThread = copyThread;
+ }
+
+ public long getPositionTimeStamp() {
+ return positionTimeStamp;
+ }
+
+ public void setPositionTimeStamp(long positionTimeStamp) {
+ this.positionTimeStamp = positionTimeStamp;
+ }
+
+ public String getInterestDB() {
+ return interestDB;
+ }
+
+ public void setInterestDB(String interestDB) {
+ this.interestDB = interestDB;
+ }
+
+ public String getInterestCollection() {
+ return interestCollection;
+ }
+
+ public void setInterestCollection(String interestCollection) {
+ this.interestCollection = interestCollection;
+ }
+
+ public String getMongoAddr() {
+ return mongoAddr;
+ }
+
+ public void setMongoAddr(String mongoAddr) {
+ this.mongoAddr = mongoAddr;
+ }
+
+ public int getMongoPort() {
+ return mongoPort;
+ }
+
+ public void setMongoPort(int mongoPort) {
+ this.mongoPort = mongoPort;
+ }
+
+ public String getMongoUserName() {
+ return mongoUserName;
+ }
+
+ public void setMongoUserName(String mongoUserName) {
+ this.mongoUserName = mongoUserName;
+ }
+
+ public String getMongoPassWord() {
+ return mongoPassWord;
+ }
+
+ public void setMongoPassWord(String mongoPassWord) {
+ this.mongoPassWord = mongoPassWord;
+ }
+
+
+ public String getDataSync() {
+ return dataSync;
+ }
+
+ public void setDataSync(String dataSync) {
+ this.dataSync = dataSync;
+ }
+
+
+ public void load(KeyValue props) {
+
+ properties2Object(props, this);
+ }
+
+ private void properties2Object(final KeyValue p, final Object object) {
+
+ Method[] methods = object.getClass().getMethods();
+ for (Method method : methods) {
+ String mn = method.getName();
+ if (mn.startsWith("set")) {
+ try {
+ String tmp = mn.substring(4);
+ String first = mn.substring(3, 4);
+
+ String key = first.toLowerCase() + tmp;
+ String property = p.getString(key);
+ if (property != null) {
+ Class<?>[] pt = method.getParameterTypes();
+ if (pt != null && pt.length > 0) {
+ String cn = pt[0].getSimpleName();
+ Object arg;
+ if (cn.equals("int") || cn.equals("Integer")) {
+ arg = Integer.parseInt(property);
+ } else if (cn.equals("long") || cn.equals("Long")) {
+ arg = Long.parseLong(property);
+ } else if (cn.equals("double") || cn.equals("Double")) {
+ arg = Double.parseDouble(property);
+ } else if (cn.equals("boolean") || cn.equals("Boolean")) {
+ arg = Boolean.parseBoolean(property);
+ } else if (cn.equals("float") || cn.equals("Float")) {
+ arg = Float.parseFloat(property);
+ } else if (cn.equals("String")) {
+ arg = property;
+ } else {
+ continue;
+ }
+ method.invoke(object, arg);
+ }
+ }
+ } catch (Throwable ignored) {
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
new file mode 100644
index 0000000..9e659c9
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
@@ -0,0 +1,61 @@
+package org.apache.connect.mongo.connector;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import io.openmessaging.connector.api.source.SourceConnector;
+import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MongoSourceConnector extends SourceConnector {
+
+ private Logger logger = LoggerFactory.getLogger(this.getClass());
+ private KeyValue keyValueConfig;
+
+
+ @Override
+ public String verifyAndSetConfig(KeyValue config) {
+ for (String requestKey : MongoReplicatorConfig.REQUEST_CONFIG) {
+ if (!config.containsKey(requestKey)) {
+ return "Request config key: " + requestKey;
+ }
+ }
+ this.keyValueConfig = config;
+ return "";
+ }
+
+ @Override
+ public void start() {
+ logger.info("start mongo source connector:{}", keyValueConfig);
+ }
+
+ @Override
+ public void stop() {
+
+ }
+
+ @Override
+ public void pause() {
+
+ }
+
+ @Override
+ public void resume() {
+
+ }
+
+ @Override
+ public Class<? extends Task> taskClass() {
+ return MongoSourceTask.class;
+ }
+
+ @Override
+ public List<KeyValue> taskConfigs() {
+ List<KeyValue> config = new ArrayList<>();
+ config.add(this.keyValueConfig);
+ return config;
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
new file mode 100644
index 0000000..e116cfb
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
@@ -0,0 +1,148 @@
+package org.apache.connect.mongo.connector;
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.data.*;
+import io.openmessaging.connector.api.source.SourceTask;
+import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.replicator.event.OperationType;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.apache.connect.mongo.replicator.MongoReplicator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class MongoSourceTask extends SourceTask {
+
+ private Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private MongoReplicator mongoReplicator;
+
+ private MongoReplicatorConfig replicatorConfig;
+
+ private String mongoSource;
+
+ @Override
+ public Collection<SourceDataEntry> poll() {
+ List<SourceDataEntry> res = new ArrayList<>();
+ ReplicationEvent event = mongoReplicator.getQueue().poll();
+ if (event == null) {
+ return new ArrayList<>();
+ }
+
+ JSONObject position = position(event);
+ Schema schema = new Schema();
+ schema.setDataSource(event.getDatabaseName());
+ schema.setName(event.getCollectionName());
+ schema.setFields(new ArrayList<>());
+ buildFieleds(schema);
+ DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
+ dataEntryBuilder.timestamp(System.currentTimeMillis())
+ .queue(event.getNamespace())
+ .entryType(event.getEntryType());
+
+ if (event.getOperationType().ordinal() == OperationType.CREATED.ordinal()) {
+ dataEntryBuilder.putFiled(Constants.CREATED, event.getDocument().toJson());
+ dataEntryBuilder.putFiled(Constants.NAMESPACE, event.getNamespace());
+ } else {
+ dataEntryBuilder.putFiled(Constants.OPERATIONTYPE, event.getOperationType().name());
+ dataEntryBuilder.putFiled(Constants.TIMESTAMP, event.getTimestamp().getValue());
+ dataEntryBuilder.putFiled(Constants.VERSION, event.getV());
+ dataEntryBuilder.putFiled(Constants.NAMESPACE, event.getNamespace());
+ dataEntryBuilder.putFiled(Constants.PATCH, event.getEventData().isPresent() ? JSONObject.toJSONString(event.getEventData().get()) : "");
+ dataEntryBuilder.putFiled(Constants.OBJECTID, event.getObjectId().isPresent() ? JSONObject.toJSONString(event.getObjectId().get()) : "");
+ }
+ SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
+ ByteBuffer.wrap(mongoSource.getBytes(StandardCharsets.UTF_8)),
+ ByteBuffer.wrap(position.toJSONString().getBytes(StandardCharsets.UTF_8)));
+ res.add(sourceDataEntry);
+ return res;
+ }
+
+ @Override
+ public void start(KeyValue config) {
+ try {
+ replicatorConfig = new MongoReplicatorConfig();
+ replicatorConfig.load(config);
+ mongoReplicator = new MongoReplicator(replicatorConfig);
+ mongoSource = new StringBuilder()
+ .append(replicatorConfig.getMongoAddr())
+ .append(replicatorConfig.getMongoPort()).toString();
+ ByteBuffer position = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(
+ mongoSource.getBytes()));
+
+ if (position != null && position.array().length > 0) {
+ String positionJson = new String(position.array(), StandardCharsets.UTF_8);
+ JSONObject jsonObject = JSONObject.parseObject(positionJson);
+ replicatorConfig.setPositionTimeStamp(jsonObject.getLongValue("timeStamp"));
+ replicatorConfig.setPositionInc(jsonObject.getIntValue("inc"));
+ } else {
+ replicatorConfig.setDataSync(Constants.INITIAL);
+ }
+ mongoReplicator.start();
+ }catch (Throwable throwable) {
+ logger.info("task start error", throwable);
+ }finally {
+ stop();
+ }
+
+
+ }
+
+ @Override
+ public void stop() {
+ logger.info("shut down.....");
+ mongoReplicator.shutdown();
+ }
+
+ @Override
+ public void pause() {
+ mongoReplicator.pause();
+ }
+
+ @Override
+ public void resume() {
+ mongoReplicator.resume();
+ }
+
+ private void buildFieleds(Schema schema) {
+ Field op = new Field(0, Constants.OPERATIONTYPE, FieldType.STRING);
+ schema.getFields().add(op);
+ Field time = new Field(1, Constants.TIMESTAMP, FieldType.INT64);
+ schema.getFields().add(time);
+ Field v = new Field(2, Constants.VERSION, FieldType.INT32);
+ schema.getFields().add(v);
+ Field namespace = new Field(3, Constants.NAMESPACE, FieldType.STRING);
+ schema.getFields().add(namespace);
+ Field operation = new Field(4, Constants.CREATED, FieldType.STRING);
+ schema.getFields().add(operation);
+ Field patch = new Field(5, Constants.PATCH, FieldType.STRING);
+ schema.getFields().add(patch);
+ Field objectId = new Field(5, Constants.OBJECTID, FieldType.STRING);
+ schema.getFields().add(objectId);
+ }
+
+ private JSONObject position(ReplicationEvent event) {
+ JSONObject jsonObject = new JSONObject();
+ switch (event.getOperationType()) {
+ case CREATED:
+ jsonObject.put(Constants.POSITION_TIMESTAMP, 0);
+ jsonObject.put(Constants.POSITION_INC, 0);
+ jsonObject.put(Constants.INITSYNC, true);
+ break;
+ default:
+ jsonObject.put(Constants.POSITION_TIMESTAMP, 0);
+ jsonObject.put(Constants.POSITION_INC, 0);
+ jsonObject.put(Constants.INITSYNC, true);
+ break;
+ }
+ return jsonObject;
+
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java b/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
new file mode 100644
index 0000000..018418c
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
@@ -0,0 +1,41 @@
+package org.apache.connect.mongo.initsync;
+
+public class CollectionMeta {
+
+ private String databaseName;
+ private String collectionName;
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public void setDatabaseName(String databaseName) {
+ this.databaseName = databaseName;
+ }
+
+ public String getCollectionName() {
+ return collectionName;
+ }
+
+ public void setCollectionName(String collectionName) {
+ this.collectionName = collectionName;
+ }
+
+ public CollectionMeta(String databaseName, String collectionName) {
+
+ this.databaseName = databaseName;
+ this.collectionName = collectionName;
+ }
+
+ public String getNameSpace() {
+ return databaseName + "." + collectionName;
+ }
+
+ @Override
+ public String toString() {
+ return "CollectionMeta{" +
+ "databaseName='" + databaseName + '\'' +
+ ", collectionName='" + collectionName + '\'' +
+ '}';
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
new file mode 100644
index 0000000..4b3a238
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
@@ -0,0 +1,146 @@
+package org.apache.connect.mongo.initsync;
+
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.MongoIterable;
+import org.apache.connect.mongo.replicator.event.DocumentConvertEvent;
+import org.apache.connect.mongo.replicator.event.OperationType;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.apache.connect.mongo.replicator.Filter;
+import org.apache.connect.mongo.replicator.MongoReplicator;
+import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.bson.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class InitSync {
+
+ private Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private MongoReplicatorConfig mongoReplicatorConfig;
+ private ExecutorService copyExecutor;
+ private MongoClient mongoClient;
+ private Filter filter;
+ private int copyThreadCount;
+ private Set<String> interestDataBases;
+ private Set<CollectionMeta> interestCollections;
+ private CountDownLatch countDownLatch;
+ private MongoReplicator mongoReplicator;
+
+ public InitSync(MongoReplicatorConfig mongoReplicatorConfig, MongoClient mongoClient, Filter filter, MongoReplicator mongoReplicator) {
+ this.mongoReplicatorConfig = mongoReplicatorConfig;
+ this.mongoClient = mongoClient;
+ this.filter = filter;
+ this.mongoReplicator = mongoReplicator;
+ init();
+ }
+
+ public void start() {
+ for (CollectionMeta collectionMeta : interestCollections) {
+ copyExecutor.submit(new CopyRunner(mongoClient, countDownLatch, collectionMeta, mongoReplicator));
+ }
+ try {
+ countDownLatch.await();
+ } catch (Exception e) {
+ } finally {
+ copyExecutor.shutdown();
+ }
+ }
+
+ private void init() {
+ interestDataBases = getInterestDataBase();
+ interestCollections = getInterestCollection(interestDataBases);
+ copyThreadCount = Math.min(interestCollections.size(), mongoReplicatorConfig.getCopyThread());
+ copyExecutor = Executors.newFixedThreadPool(copyThreadCount, new ThreadFactory() {
+
+ AtomicInteger threads = new AtomicInteger();
+
+ @Override
+ public Thread newThread(Runnable r) {
+ return new Thread(r, "copy_collection_thread_" + threads.incrementAndGet());
+ }
+ });
+ countDownLatch = new CountDownLatch(interestCollections.size());
+ }
+
+ private Set<CollectionMeta> getInterestCollection(Set<String> interestDataBases) {
+ Set<CollectionMeta> res = new HashSet<>();
+ for (String interestDataBase : interestDataBases) {
+ MongoIterable<String> collectionNames = mongoClient.getDatabase(interestDataBase).listCollectionNames();
+ MongoCursor<String> iterator = collectionNames.iterator();
+ while (iterator.hasNext()) {
+ String collectionName = iterator.next();
+ if (filter.filterCollectionName(collectionName)) {
+ CollectionMeta collectionMeta = new CollectionMeta(interestDataBase, collectionName);
+ res.add(collectionMeta);
+ }
+ }
+ }
+
+ return res;
+
+ }
+
+ private Set<String> getInterestDataBase() {
+ Set<String> res = new HashSet<>();
+ MongoIterable<String> databaseNames = mongoClient.listDatabaseNames();
+ MongoCursor<String> iterator = databaseNames.iterator();
+ while (iterator.hasNext()) {
+ String dataBaseName = iterator.next();
+ if (filter.filterDatabaseName(dataBaseName)) {
+ res.add(dataBaseName);
+ }
+ }
+
+ return res;
+ }
+
+ class CopyRunner implements Runnable {
+
+ private MongoClient mongoClient;
+ private CountDownLatch countDownLatch;
+ private CollectionMeta collectionMeta;
+ private MongoReplicator mongoReplicator;
+
+ public CopyRunner(MongoClient mongoClient, CountDownLatch countDownLatch, CollectionMeta collectionMeta, MongoReplicator mongoReplicator) {
+ this.mongoClient = mongoClient;
+ this.countDownLatch = countDownLatch;
+ this.collectionMeta = collectionMeta;
+ this.mongoReplicator = mongoReplicator;
+ }
+
+ @Override
+ public void run() {
+
+ try {
+
+ MongoCursor<Document> mongoCursor = mongoClient.getDatabase(collectionMeta.getDatabaseName())
+ .getCollection(collectionMeta.getCollectionName())
+ .find()
+ .batchSize(200)
+ .iterator();
+
+ while (mongoReplicator.isRuning() && mongoCursor.hasNext()) {
+ Document document = mongoCursor.next();
+ ReplicationEvent event = DocumentConvertEvent.convert(document);
+ event.setOperationType(OperationType.CREATED);
+ event.setNamespace(collectionMeta.getNameSpace());
+ mongoReplicator.publishEvent(event);
+ }
+ } finally {
+ countDownLatch.countDown();
+ }
+ logger.info("database:{}, collection:{}, init sync done", collectionMeta.getDatabaseName(), collectionMeta.getCollectionName());
+ }
+ }
+
+}
+
+
+
+
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Filter.java b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
new file mode 100644
index 0000000..05dec54
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
@@ -0,0 +1,74 @@
+package org.apache.connect.mongo.replicator;
+
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.replicator.event.OperationType;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+
+import java.util.function.Function;
+
+public class Filter {
+
+ private Function<String, Boolean> dbFilter;
+ private Function<String, Boolean> collectionFilter;
+ private Function<OperationType, Boolean> noopFilter;
+
+
+ public Filter(MongoReplicatorConfig mongoReplicatorConfig) {
+ if (StringUtils.isNotBlank(mongoReplicatorConfig.getInterestDB())) {
+ dbFilter = (dataBaseName) -> {
+ if (StringUtils.isBlank(dataBaseName)) {
+ return true;
+ }
+ String interestDB = mongoReplicatorConfig.getInterestDB();
+ String[] db = StringUtils.split(interestDB, ",");
+ if (ArrayUtils.contains(db, dataBaseName)) {
+ return true;
+ }
+
+ return false;
+ };
+ } else {
+ dbFilter = (dataBaseName) -> true;
+ }
+
+
+ if (StringUtils.isNotBlank(mongoReplicatorConfig.getInterestCollection())) {
+ collectionFilter = (collectionName) -> {
+ if (StringUtils.isBlank(collectionName)) {
+ return true;
+ }
+
+ String interestCollection = mongoReplicatorConfig.getInterestCollection();
+ String[] coll = StringUtils.split(interestCollection, ",");
+ if (ArrayUtils.contains(coll, collectionName)) {
+ return true;
+ }
+ return false;
+ };
+ } else {
+ collectionFilter = (collectionName) -> true;
+ }
+
+ noopFilter = (opeartionType) -> opeartionType.ordinal() != OperationType.NOOP.ordinal();
+ }
+
+
+ public boolean filterDatabaseName(String dataBaseName) {
+ return dbFilter.apply(dataBaseName);
+ }
+
+
+ public boolean filterCollectionName(String collectionName) {
+ return collectionFilter.apply(collectionName);
+ }
+
+ public boolean filterEvent(ReplicationEvent event) {
+ return dbFilter.apply(event.getDatabaseName())
+ && collectionFilter.apply(event.getCollectionName())
+ && noopFilter.apply(event.getOperationType());
+ }
+
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
new file mode 100644
index 0000000..326ba03
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
@@ -0,0 +1,152 @@
+package org.apache.connect.mongo.replicator;
+
+import com.mongodb.*;
+import com.mongodb.client.*;
+import com.mongodb.client.MongoClient;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
+import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.connect.mongo.replicator.Constants.*;
+
+
+public class MongoReplicator {
+
+ private Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private AtomicBoolean running = new AtomicBoolean();
+
+ private MongoReplicatorConfig mongoReplicatorConfig;
+
+ private MongoClientSettings clientSettings;
+
+ private ConnectionString connectionString;
+
+ private MongoClient mongoClient;
+
+ private BlockingQueue<ReplicationEvent> queue = new LinkedBlockingQueue<>();
+
+ private Filter filter;
+
+ private ExecutorService executorService;
+
+ private volatile boolean pause = false;
+
+ public MongoReplicator(MongoReplicatorConfig mongoReplicatorConfig) {
+ this.mongoReplicatorConfig = mongoReplicatorConfig;
+ this.filter = new Filter(mongoReplicatorConfig);
+ this.executorService = Executors.newSingleThreadExecutor((r) ->new Thread(r, "real_time_replica_thread"));
+
+ buildConnectionString();
+ }
+
+ public void start() {
+
+ try {
+ if (!running.compareAndSet(false, true)) {
+ logger.info("the java mongo replica already start");
+ return;
+ }
+
+ this.clientSettings = MongoClientSettings.builder().applicationName(APPLICATION_NAME)
+ .applyConnectionString(connectionString)
+ .retryWrites(true)
+ .build();
+ this.mongoClient = MongoClients.create(clientSettings);
+ this.isReplicaMongo();
+ executorService.submit(new ReplicatorTask(this, mongoClient, mongoReplicatorConfig, filter));
+ }catch (Exception e) {
+ logger.info("start replicator error", e);
+ }finally {
+ shutdown();
+ }
+
+ }
+
+
+ private void buildConnectionString() {
+ checkConfig();
+ StringBuilder sb = new StringBuilder();
+ sb.append("mongodb://");
+ if (StringUtils.isNotBlank(mongoReplicatorConfig.getMongoUserName())
+ && StringUtils.isNotBlank(mongoReplicatorConfig.getMongoPassWord())) {
+ sb.append(mongoReplicatorConfig.getMongoUserName());
+ sb.append(":");
+ sb.append(mongoReplicatorConfig.getMongoPassWord());
+ sb.append("@");
+
+ }
+ sb.append(mongoReplicatorConfig.getMongoAddr());
+ sb.append(":");
+ sb.append(mongoReplicatorConfig.getMongoPort());
+
+ this.connectionString = new ConnectionString(sb.toString());
+ }
+
+ private void checkConfig() {
+ Validate.notBlank(mongoReplicatorConfig.getMongoAddr(), "mongo url is blank");
+ Validate.isTrue(mongoReplicatorConfig.getMongoPort() > 0 && mongoReplicatorConfig.getMongoPort() < 65535, "mongo port should >0 and <65535");
+
+ }
+
+ private boolean isReplicaMongo() {
+ MongoDatabase local = mongoClient.getDatabase(MONGO_LOCAL_DATABASE);
+ MongoIterable<String> collectionNames = local.listCollectionNames();
+ for (String collectionName : collectionNames) {
+ if (MONGO_OPLOG_RS.equals(collectionName)) {
+ return true;
+ }
+ }
+ this.shutdown();
+ throw new IllegalStateException(String.format("url:%s, port:%s is not replica", mongoReplicatorConfig.getMongoAddr(), mongoReplicatorConfig.getMongoPort()));
+ }
+
+ public void shutdown() {
+ if (running.compareAndSet(true, false)) {
+ if (!this.executorService.isShutdown()) {
+ executorService.shutdown();
+ }
+ if (this.mongoClient != null) {
+ this.mongoClient.close();
+ }
+ }
+
+ }
+
+ public void publishEvent(ReplicationEvent replicationEvent) {
+ while (true) {
+ try {
+ queue.put(replicationEvent);
+ break;
+ } catch (Exception e) {
+ }
+ }
+ }
+
+
+ public void pause() {
+ pause = true;
+ }
+
+ public void resume() {
+ pause = false;
+ }
+
+ public boolean isPause() {
+ return pause;
+ }
+
+ public boolean isRuning() {
+ return running.get();
+ }
+
+ public BlockingQueue<ReplicationEvent> getQueue() {
+ return queue;
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
new file mode 100644
index 0000000..9e4c67b
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
@@ -0,0 +1,84 @@
+package org.apache.connect.mongo.replicator;
+
+import com.mongodb.CursorType;
+import com.mongodb.client.FindIterable;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Filters;
+import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.replicator.event.DocumentConvertEvent;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.apache.connect.mongo.initsync.InitSync;
+import org.bson.Document;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class ReplicatorTask implements Runnable {
+
+ private Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private MongoReplicator mongoReplicator;
+
+ private MongoClient mongoClient;
+
+ private MongoReplicatorConfig mongoReplicatorConfig;
+
+ private Filter filter;
+
+ public ReplicatorTask(MongoReplicator mongoReplicator, MongoClient mongoClient, MongoReplicatorConfig mongoReplicatorConfig, Filter filter) {
+ this.mongoReplicator = mongoReplicator;
+ this.mongoReplicatorConfig = mongoReplicatorConfig;
+ this.mongoClient = mongoClient;
+ this.filter = filter;
+ }
+
+ @Override
+ public void run() {
+
+ if (Constants.INITIAL.equals(mongoReplicatorConfig.getDataSync())) {
+ InitSync initSync = new InitSync(mongoReplicatorConfig, mongoClient, filter, mongoReplicator);
+ initSync.start();
+ }
+
+ MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
+ FindIterable<Document> iterable;
+ if (mongoReplicatorConfig.getPositionTimeStamp() > 0 && mongoReplicatorConfig.getPositionTimeStamp() < System.currentTimeMillis()) {
+ iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find(
+ Filters.gt("ts", mongoReplicatorConfig.getPositionTimeStamp()));
+ } else {
+ iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
+ }
+ MongoCursor<Document> cursor = iterable.sort(new Document("$natural", 1))
+ .noCursorTimeout(true)
+ .cursorType(CursorType.TailableAwait)
+ .batchSize(200)
+ .iterator();
+
+ while (mongoReplicator.isRuning()) {
+ try {
+ executorCursor(cursor);
+ Thread.sleep(100);
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ logger.error("mongoReplicator shutdown.....");
+ mongoReplicator.shutdown();
+ }
+ }
+ }
+
+
+ private void executorCursor(MongoCursor<Document> cursor) {
+ while (cursor.hasNext() && !mongoReplicator.isPause()) {
+ Document document = cursor.next();
+ ReplicationEvent event = DocumentConvertEvent.convert(document);
+ if (filter.filterEvent(event)) {
+ mongoReplicator.publishEvent(event);
+ }
+ }
+ }
+
+
+}
diff --git a/src/test/java/org/apache/connect/mongo/FilterTest.java b/src/test/java/org/apache/connect/mongo/FilterTest.java
new file mode 100644
index 0000000..24f36d9
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/FilterTest.java
@@ -0,0 +1,67 @@
+package org.apache.connect.mongo;
+
+import org.apache.connect.mongo.replicator.Filter;
+import org.apache.connect.mongo.replicator.event.OperationType;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class FilterTest {
+
+
+ private MongoReplicatorConfig config;
+
+
+ @Before
+ public void init() {
+ config = new MongoReplicatorConfig();
+
+ }
+
+ @Test
+ public void testSpecialDb() {
+ config.setInterestDB("test,admin");
+ Filter filter = new Filter(config);
+ Assert.assertTrue(filter.filterDatabaseName("test"));
+ Assert.assertFalse(filter.filterDatabaseName("test01"));
+ }
+
+
+ @Test
+ public void testBlankDb() {
+ Filter filter = new Filter(config);
+ Assert.assertTrue(filter.filterDatabaseName("test"));
+ Assert.assertTrue(filter.filterDatabaseName("test01"));
+ }
+
+
+ @Test
+ public void testSpecialCollection() {
+ config.setInterestCollection("test,admin");
+ Filter filter = new Filter(config);
+ Assert.assertTrue(filter.filterCollectionName("test"));
+ Assert.assertFalse(filter.filterCollectionName("test01"));
+ }
+
+
+ @Test
+ public void testBlankCollection() {
+ Filter filter = new Filter(config);
+ Assert.assertTrue(filter.filterCollectionName("test"));
+ Assert.assertTrue(filter.filterCollectionName("test01"));
+ }
+
+
+ @Test
+ public void testFilterEvent() {
+ Filter filter = new Filter(config);
+ ReplicationEvent replicationEvent = new ReplicationEvent();
+ replicationEvent.setOperationType(OperationType.NOOP);
+ Assert.assertFalse(filter.filterEvent(replicationEvent));
+ replicationEvent.setOperationType(OperationType.DBCOMMAND);
+ Assert.assertTrue(filter.filterEvent(replicationEvent));
+ }
+
+
+}
diff --git a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
new file mode 100644
index 0000000..04e1374
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
@@ -0,0 +1,45 @@
+package org.apache.connect.mongo;
+
+import io.openmessaging.internal.DefaultKeyValue;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.connector.MongoSourceConnector;
+import org.apache.connect.mongo.connector.MongoSourceTask;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MongoSourceConnectorTest {
+
+ private MongoSourceConnector mongoSourceConnector;
+ private DefaultKeyValue keyValue;
+
+ @Before
+ public void before() {
+ mongoSourceConnector = new MongoSourceConnector();
+ keyValue = new DefaultKeyValue();
+ }
+
+ @Test
+ public void takeClass() {
+ Assert.assertEquals(mongoSourceConnector.taskClass(), MongoSourceTask.class);
+ }
+
+
+ @Test
+ public void verifyConfig() {
+ keyValue.put("mongoAddr", "127.0.0.1");
+ String s = mongoSourceConnector.verifyAndSetConfig(keyValue);
+ Assert.assertTrue(s.contains("Request config key:"));
+ keyValue.put("mongoPort", 27017);
+ s = mongoSourceConnector.verifyAndSetConfig(keyValue);
+ Assert.assertTrue(StringUtils.isBlank(s));
+ }
+
+
+
+
+
+
+
+
+}
diff --git a/src/test/java/org/apache/connect/mongo/ReplicatorTest.java b/src/test/java/org/apache/connect/mongo/ReplicatorTest.java
new file mode 100644
index 0000000..b0319f1
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/ReplicatorTest.java
@@ -0,0 +1,33 @@
+package org.apache.connect.mongo;
+
+import org.apache.connect.mongo.replicator.MongoReplicator;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ReplicatorTest {
+
+ private MongoReplicatorConfig config;
+
+ @Before
+ public void before() {
+ config = new MongoReplicatorConfig();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testNoPort() {
+ config.setMongoAddr("127.0.0.1");
+ MongoReplicator mongoReplicator = new MongoReplicator(config);
+ mongoReplicator.start();
+ }
+
+
+ @Test
+ public void testNoPort1() {
+ config.setMongoAddr("127.0.0.1");
+ config.setMongoPort(27012);
+ MongoReplicator mongoReplicator = new MongoReplicator(config);
+ mongoReplicator.start();
+ }
+
+
+}
[rocketmq-connect] 12/13: bump up the connect version to 0.1.1
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit b307466f3fc4fa25ba21de87c568a4095cf18f4f
Author: Heng Du <du...@apache.org>
AuthorDate: Mon Mar 30 11:43:35 2020 +0800
bump up the connect version to 0.1.1
bump up the connect version to 0.1.1
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 7ea7ab1..5c412bf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -153,7 +153,7 @@
<dependency>
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-connector</artifactId>
- <version>0.1.0-beta</version>
+ <version>0.1.1</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
[rocketmq-connect] 11/13: clean code
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit db51f7ff808a10209ff7eafb68cec0e08f25f058
Author: 李平 <17...@qq.com>
AuthorDate: Mon Aug 26 14:20:51 2019 +0800
clean code
---
README.md | 17 +++++++--
.../apache/connect/mongo/initsync/InitSync.java | 3 +-
.../{Filter.java => OperationFilter.java} | 4 +-
.../mongo/replicator/ReplicaSetManager.java | 2 +-
.../mongo/replicator/ReplicaSetsContext.java | 10 ++---
.../connect/mongo/replicator/ReplicatorTask.java | 25 ++++++------
.../java/org/apache/connect/mongo/FilterTest.java | 26 ++++++-------
.../org/apache/connect/mongo/MongoFactoryTest.java | 44 ++++++++++++----------
8 files changed, 73 insertions(+), 58 deletions(-)
diff --git a/README.md b/README.md
index e32c455..bec30a3 100644
--- a/README.md
+++ b/README.md
@@ -2,14 +2,18 @@
this is source connector moudle for mongo,you can run this by running rocketmq connecotr api,
-some junit rely on mongo database you can start with a docker container
+some junit rely on mongo database you can flow step run a mongo container
-`docker run -p27027:27017 --name mongo-test -d mongo:4.0.10 --replSet "repl1"`
+- `docker run -p27027:27017 --name mongo-test -d mongo:4.0.10 --replSet "repl1"`
+- `docker exec -it mongo-test mongo `
+- `rs.initiate()`
-and then init a mongo replicaSet
+init a mongo replicaSet run all junit test
-`docker exec -it mongo-test mongo ` and `rs.initiate()` and then you can run all junit test
+## a special junit
+method `MongoFactoryTest#testSSLTrustStore` is for mongo ssl or tsl test,need mongod config ssl mode， if you want use ssl or tsl you need
+modify junit , appoint ssl or tsl pem path and password。
## task config params
@@ -32,3 +36,8 @@ and then init a mongo replicaSet
| zlibCompressionLevel | zlib compressors level| int (1-7)|
| trustStore | ssl pem| path|
| trustStorePassword | ssl pem decrypt password | string|
+
+
+## use case
+
+`http://127.0.0.1:8081/connectors/testMongoReplicaSet?config={"connector-class":"org.apache.connect.mongo.connector.MongoSourceConnector","oms-driver-url":"oms:rocketmq://localhost:9876/default:default","mongoAddr":"rep1/127.0.0.1:27077,127.0.0.1:27078,127.0.0.1:27080","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}`
diff --git a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
index 3d68fac..a51b727 100644
--- a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
+++ b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
@@ -65,7 +65,8 @@ public class InitSync {
}
try {
countDownLatch.await();
- } catch (Exception e) {
+ } catch (InterruptedException e) {
+ logger.error("init sync wait countDownLatch interrupted");
} finally {
copyExecutor.shutdown();
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Filter.java b/src/main/java/org/apache/connect/mongo/replicator/OperationFilter.java
similarity index 96%
rename from src/main/java/org/apache/connect/mongo/replicator/Filter.java
rename to src/main/java/org/apache/connect/mongo/replicator/OperationFilter.java
index a517822..a173f6c 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/Filter.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/OperationFilter.java
@@ -29,13 +29,13 @@ import org.apache.connect.mongo.initsync.CollectionMeta;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
-public class Filter {
+public class OperationFilter {
private Function<CollectionMeta, Boolean> dbAndCollectionFilter;
private Map<String, List<String>> interestMap = new HashMap<>();
private Function<OperationType, Boolean> notNoopFilter;
- public Filter(SourceTaskConfig sourceTaskConfig) {
+ public OperationFilter(SourceTaskConfig sourceTaskConfig) {
String interestDbAndCollection = sourceTaskConfig.getInterestDbAndCollection();
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
index c5757d8..88097d8 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetManager.java
@@ -75,7 +75,7 @@ public class ReplicaSetManager {
}
private void validate() {
- Validate.isTrue(replicaConfigByName.size() > 0, "task config mongoAdd need special replicaSet addr");
+ Validate.isTrue(replicaConfigByName.size() > 0, "task config mongoAddr need special replicaSet addr");
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
index b067256..8dd85d7 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
@@ -42,18 +42,16 @@ public class ReplicaSetsContext {
private AtomicBoolean initSyncAbort = new AtomicBoolean();
- private Filter filter;
+ private OperationFilter operationFilter;
private MongoClientFactory mongoClientFactory;
- private Map<String, Position> lastPositionMap;
public ReplicaSetsContext(SourceTaskConfig taskConfig) {
this.taskConfig = taskConfig;
this.replicaSets = new ArrayList<>();
- this.lastPositionMap = new HashMap<>();
this.dataEntryQueue = new LinkedBlockingDeque<>();
- this.filter = new Filter(taskConfig);
+ this.operationFilter = new OperationFilter(taskConfig);
this.mongoClientFactory = new MongoClientFactory(taskConfig);
}
@@ -62,11 +60,11 @@ public class ReplicaSetsContext {
}
public boolean filterEvent(ReplicationEvent event) {
- return filter.filterEvent(event);
+ return operationFilter.filterEvent(event);
}
public boolean filterMeta(CollectionMeta collectionMeta) {
- return filter.filterMeta(collectionMeta);
+ return operationFilter.filterMeta(collectionMeta);
}
public int getCopyThread() {
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
index 4c142ce..cd78f24 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
@@ -54,20 +54,23 @@ public class ReplicatorTask implements Runnable {
@Override
public void run() {
- BsonTimestamp firstAvailablePosition = findOplogFirstPosition();
-
- // inValid or
- // user config dataSync or
- // user config or runtime saved position lt first oplog position maybe some operation is lost so need dataSync
- if (!replicaSetConfig.getPosition().isValid() || replicaSetConfig.getPosition().isInitSync()
- || replicaSetConfig.getPosition().converBsonTimeStamp().compareTo(firstAvailablePosition) < 0) {
- recordOplogLastPosition();
+ BsonTimestamp firstAvailablePosition = findFirstOplogPosition();
+
+ Position userConfigOrRuntimePosition = replicaSetConfig.getPosition();
+
+ boolean needDataSync = !userConfigOrRuntimePosition.isValid()
+ || userConfigOrRuntimePosition.isInitSync()
+ // userConfigOrRuntimePosition.position < firstAvailablePosition maybe lost some operations
+ || userConfigOrRuntimePosition.converBsonTimeStamp().compareTo(firstAvailablePosition) < 0;
+
+ if (needDataSync) {
+ recordLastOplogPosition();
InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
initSync.start();
}
- if (!replicaSet.isRuning() || !replicaSetsContext.isInitSyncAbort()) {
+ if (!replicaSet.isRuning() || replicaSetsContext.isInitSyncAbort()) {
return;
}
@@ -96,7 +99,7 @@ public class ReplicatorTask implements Runnable {
logger.info("replicaSet:{}, already shutdown, replicaTask end of life cycle", replicaSetConfig);
}
- private BsonTimestamp findOplogFirstPosition() {
+ private BsonTimestamp findFirstOplogPosition() {
MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
Document lastOplog = iterable.sort(new Document("$natural", 1)).limit(1).first();
@@ -104,7 +107,7 @@ public class ReplicatorTask implements Runnable {
return timestamp;
}
- private void recordOplogLastPosition() {
+ private void recordLastOplogPosition() {
MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
FindIterable<Document> iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
Document lastOplog = iterable.sort(new Document("$natural", -1)).limit(1).first();
diff --git a/src/test/java/org/apache/connect/mongo/FilterTest.java b/src/test/java/org/apache/connect/mongo/FilterTest.java
index 31c8f4d..d5deefd 100644
--- a/src/test/java/org/apache/connect/mongo/FilterTest.java
+++ b/src/test/java/org/apache/connect/mongo/FilterTest.java
@@ -6,7 +6,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.connect.mongo.initsync.CollectionMeta;
-import org.apache.connect.mongo.replicator.Filter;
+import org.apache.connect.mongo.replicator.OperationFilter;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
import org.junit.Assert;
@@ -30,16 +30,16 @@ public class FilterTest {
collections.add("person");
insterest.put("test", collections);
sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
- Filter filter = new Filter(sourceTaskConfig);
- Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "person")));
- Assert.assertFalse(filter.filterMeta(new CollectionMeta("test", "person01")));
+ OperationFilter operationFilter = new OperationFilter(sourceTaskConfig);
+ Assert.assertTrue(operationFilter.filterMeta(new CollectionMeta("test", "person")));
+ Assert.assertFalse(operationFilter.filterMeta(new CollectionMeta("test", "person01")));
}
@Test
public void testBlankDb() {
- Filter filter = new Filter(sourceTaskConfig);
- Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "test")));
- Assert.assertTrue(filter.filterMeta(new CollectionMeta("test1", "test01")));
+ OperationFilter operationFilter = new OperationFilter(sourceTaskConfig);
+ Assert.assertTrue(operationFilter.filterMeta(new CollectionMeta("test", "test")));
+ Assert.assertTrue(operationFilter.filterMeta(new CollectionMeta("test1", "test01")));
}
@Test
@@ -48,19 +48,19 @@ public class FilterTest {
collections.add("*");
insterest.put("test", collections);
sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
- Filter filter = new Filter(sourceTaskConfig);
- Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "testsad")));
- Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "tests032")));
+ OperationFilter operationFilter = new OperationFilter(sourceTaskConfig);
+ Assert.assertTrue(operationFilter.filterMeta(new CollectionMeta("test", "testsad")));
+ Assert.assertTrue(operationFilter.filterMeta(new CollectionMeta("test", "tests032")));
}
@Test
public void testFilterEvent() {
- Filter filter = new Filter(sourceTaskConfig);
+ OperationFilter operationFilter = new OperationFilter(sourceTaskConfig);
ReplicationEvent replicationEvent = new ReplicationEvent();
replicationEvent.setOperationType(OperationType.NOOP);
- Assert.assertFalse(filter.filterEvent(replicationEvent));
+ Assert.assertFalse(operationFilter.filterEvent(replicationEvent));
replicationEvent.setOperationType(OperationType.DB_COMMAND);
- Assert.assertTrue(filter.filterEvent(replicationEvent));
+ Assert.assertTrue(operationFilter.filterEvent(replicationEvent));
}
diff --git a/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java b/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
index 0f02064..e47d2c4 100644
--- a/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
@@ -2,12 +2,16 @@ package org.apache.connect.mongo;
import com.mongodb.MongoClientSettings;
import com.mongodb.MongoTimeoutException;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoCursor;
import com.mongodb.client.internal.MongoClientImpl;
import java.lang.reflect.Field;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.StringUtils;
import org.apache.connect.mongo.replicator.MongoClientFactory;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.bson.Document;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -149,25 +153,25 @@ public class MongoFactoryTest {
return null;
}
-// @Test
-// public void testSSLTrustStore() {
-// sourceTaskConfig.setMongoUserName("user_test");
-// sourceTaskConfig.setMongoPassWord("pwd_test");
-// sourceTaskConfig.setSsl("ssl");
-// sourceTaskConfig.setSslInvalidHostNameAllowed("true");
-// sourceTaskConfig.setTrustStore("/Users/liping/test.pem");
-// sourceTaskConfig.setTrustStorePassword("test001");
-// sourceTaskConfig.setServerSelectionTimeoutMS("10000");
-// MongoClient client = mongoClientFactory.createMongoClient(replicaSetConfig);
-// MongoCollection<Document> collection = client.getDatabase("test").getCollection("person");
-// Document document = new Document();
-// document.put("name", "liping");
-// collection.insertOne(document);
-// MongoCursor<Document> iterator = collection.find().iterator();
-// while (iterator.hasNext()) {
-// System.out.println(iterator.next());
-// }
-//
-// }
+ @Test
+ public void testSSLTrustStore() {
+ sourceTaskConfig.setMongoUserName("user_test");
+ sourceTaskConfig.setMongoPassWord("pwd_test");
+ sourceTaskConfig.setSsl(true);
+ sourceTaskConfig.setSslInvalidHostNameAllowed(true);
+ sourceTaskConfig.setTrustStore("/Users/home/test.pem");
+ sourceTaskConfig.setTrustStorePassword("test001");
+ sourceTaskConfig.setServerSelectionTimeoutMS(10000);
+ MongoClient client = mongoClientFactory.createMongoClient(replicaSetConfig);
+ MongoCollection<Document> collection = client.getDatabase("test").getCollection("person");
+ Document document = new Document();
+ document.put("name", "test");
+ collection.insertOne(document);
+ MongoCursor<Document> iterator = collection.find().iterator();
+ while (iterator.hasNext()) {
+ System.out.println(iterator.next());
+ }
+
+ }
}
[rocketmq-connect] 08/13: fix test verifyConfig
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 98da0fbcca9eda5824d37bb45288de41d3818f5a
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Tue Aug 13 17:01:03 2019 +0800
fix test verifyConfig
---
src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
index d921e63..1f6227b 100644
--- a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
@@ -45,9 +45,8 @@ public class MongoSourceConnectorTest {
@Test
public void verifyConfig() {
- keyValue.put("mongoAddr", "shardName=replicaName:127.0.0.1:27017");
String s = mongoSourceConnector.verifyAndSetConfig(keyValue);
- Assert.assertTrue(s.contains("Request sourceTaskConfig key:"));
+ Assert.assertTrue(s.contains("Request config key:"));
}
[rocketmq-connect] 06/13: fix bug
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 6777f2317f083112cadd652270fd141290122f1b
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Thu Aug 8 19:53:39 2019 +0800
fix bug
---
.../org/apache/connect/mongo/MongoReplicatorConfig.java | 6 +++---
.../apache/connect/mongo/connector/MongoSourceTask.java | 11 ++++++-----
.../apache/connect/mongo/replicator/MongoReplicator.java | 15 ++++++++++-----
.../apache/connect/mongo/replicator/ReplicatorTask.java | 2 +-
src/test/java/org/apache/connect/mongo/MongoTest.java | 10 +++++-----
5 files changed, 25 insertions(+), 19 deletions(-)
diff --git a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java b/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
index b7044ec..9097640 100644
--- a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
+++ b/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
@@ -17,7 +17,7 @@ public class MongoReplicatorConfig {
private String interestDbAndCollection;
private int positionTimeStamp;
private int positionInc;
- private String dataSync;
+ private boolean dataSync;
private int copyThread = Runtime.getRuntime().availableProcessors();
@@ -85,11 +85,11 @@ public class MongoReplicatorConfig {
}
- public String getDataSync() {
+ public boolean getDataSync() {
return dataSync;
}
- public void setDataSync(String dataSync) {
+ public void setDataSync(boolean dataSync) {
this.dataSync = dataSync;
}
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
index 9176ab7..7272878 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
@@ -4,11 +4,11 @@ import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.data.*;
import io.openmessaging.connector.api.source.SourceTask;
-import org.apache.connect.mongo.replicator.Constants;
import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.replicator.MongoReplicator;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
-import org.apache.connect.mongo.replicator.MongoReplicator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,7 +44,7 @@ public class MongoSourceTask extends SourceTask {
buildFieleds(schema);
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(event.getNamespace().replace(".", "-"))
+ .queue(event.getNamespace().replace(".", "-").replace("$", ""))
.entryType(event.getEntryType());
if (event.getOperationType().ordinal() == OperationType.CREATED.ordinal()) {
@@ -78,10 +78,11 @@ public class MongoSourceTask extends SourceTask {
if (position != null && position.array().length > 0) {
String positionJson = new String(position.array(), StandardCharsets.UTF_8);
JSONObject jsonObject = JSONObject.parseObject(positionJson);
- replicatorConfig.setPositionTimeStamp(jsonObject.getLongValue("timeStamp"));
+ replicatorConfig.setPositionTimeStamp(jsonObject.getIntValue("timeStamp"));
replicatorConfig.setPositionInc(jsonObject.getIntValue("inc"));
+ replicatorConfig.setDataSync(jsonObject.getBooleanValue(Constants.INITSYNC));
} else {
- replicatorConfig.setDataSync(Constants.INITIAL);
+ replicatorConfig.setDataSync(true);
}
mongoReplicator.start();
}catch (Throwable throwable) {
diff --git a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
index 0f8520b..60b8d3d 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
@@ -1,16 +1,21 @@
package org.apache.connect.mongo.replicator;
-import com.mongodb.*;
-import com.mongodb.client.*;
+import com.mongodb.ConnectionString;
+import com.mongodb.MongoClientSettings;
import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.MongoIterable;
import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.Validate;
import org.apache.connect.mongo.MongoReplicatorConfig;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.concurrent.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.connect.mongo.replicator.Constants.*;
@@ -81,7 +86,7 @@ public class MongoReplicator {
}
sb.append(mongoReplicatorConfig.getMongoAddr());
sb.append("/");
- if (StringUtils.isBlank(mongoReplicatorConfig.getReplicaSet())) {
+ if (StringUtils.isNotBlank(mongoReplicatorConfig.getReplicaSet())) {
sb.append("?");
sb.append("replicaSet=");
sb.append(mongoReplicatorConfig.getReplicaSet());
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
index 32bb25b..766225f 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
@@ -37,7 +37,7 @@ public class ReplicatorTask implements Runnable {
@Override
public void run() {
- if (Constants.INITIAL.equals(mongoReplicatorConfig.getDataSync())) {
+ if (mongoReplicatorConfig.getDataSync()) {
InitSync initSync = new InitSync(mongoReplicatorConfig, mongoClient, filter, mongoReplicator);
initSync.start();
}
diff --git a/src/test/java/org/apache/connect/mongo/MongoTest.java b/src/test/java/org/apache/connect/mongo/MongoTest.java
index 849c00c..cc83fbe 100644
--- a/src/test/java/org/apache/connect/mongo/MongoTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoTest.java
@@ -2,10 +2,10 @@ package org.apache.connect.mongo;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.ConnectionString;
-import com.mongodb.CursorType;
import com.mongodb.MongoClientSettings;
-import com.mongodb.client.*;
-import com.mongodb.client.model.Filters;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoCollection;
import io.openmessaging.connector.api.data.EntryType;
import org.apache.connect.mongo.initsync.InitSync;
import org.apache.connect.mongo.replicator.Constants;
@@ -36,7 +36,7 @@ public class MongoTest {
@Before
public void before() {
MongoClientSettings.Builder builder = MongoClientSettings.builder();
- builder.applyConnectionString(new ConnectionString("mongodb://127.0.0.1:27077"));
+ builder.applyConnectionString(new ConnectionString("mongodb://127.0.0.1:27018"));
mongoClient = MongoClients.create(builder.build());
}
@@ -68,7 +68,7 @@ public class MongoTest {
public void testInitSyncCopy() throws NoSuchFieldException, IllegalAccessException, InterruptedException {
MongoCollection<Document> collection = mongoClient.getDatabase("test").getCollection("person");
collection.deleteMany(new Document());
- int count = 100;
+ int count = 1;
List<Document> documents = new ArrayList<>(count);
for (int i = 0; i < count; i++) {
Document document = new Document();
[rocketmq-connect] 02/13: add junit test and modify some code
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 6ca400a1e37743bf0a701e49bd5b3b28efeba9b7
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Mon Aug 5 15:02:53 2019 +0800
add junit test and modify some code
---
.../apache/connect/mongo/replicator/Constants.java | 33 +++++
.../replicator/event/DocumentConvertEvent.java | 35 +++++
.../mongo/replicator/event/OperationType.java | 28 ++++
.../mongo/replicator/event/ReplicationEvent.java | 153 +++++++++++++++++++++
4 files changed, 249 insertions(+)
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Constants.java b/src/main/java/org/apache/connect/mongo/replicator/Constants.java
new file mode 100644
index 0000000..668fd91
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/Constants.java
@@ -0,0 +1,33 @@
+package org.apache.connect.mongo.replicator;
+
+public class Constants {
+
+
+ public static final String APPLICATION_NAME = "java-mongo-replicator";
+
+
+ public static final String MONGO_LOCAL_DATABASE = "local";
+ public static final String MONGO_OPLOG_RS = "oplog.rs";
+
+
+
+ public static final String OPERATIONTYPE = "op";
+ public static final String TIMESTAMP = "ts";
+ public static final String VERSION = "v";
+ public static final String HASH = "h";
+ public static final String NAMESPACE = "ns";
+ public static final String OPERATION = "o";
+ public static final String OBJECTID = "o2";
+
+
+ public static final String CREATED = "created";
+ public static final String PATCH = "patch";
+
+ public static final String INITIAL = "initial";
+
+
+ public static final String POSITION_TIMESTAMP = "timeStamp";
+ public static final String POSITION_INC = "inc";
+ public static final String INITSYNC = "initSync";
+
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java b/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java
new file mode 100644
index 0000000..6b902e7
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java
@@ -0,0 +1,35 @@
+package org.apache.connect.mongo.replicator.event;
+
+import org.bson.BsonTimestamp;
+import org.bson.Document;
+
+import java.util.Optional;
+
+import static org.apache.connect.mongo.replicator.Constants.*;
+
+
+public class DocumentConvertEvent {
+
+ public static ReplicationEvent convert(Document document) {
+ ReplicationEvent event = null;
+ try {
+
+ OperationType operationType = OperationType.getOperationType(document.getString(OPERATIONTYPE));
+ BsonTimestamp timestamp = (BsonTimestamp) document.get(TIMESTAMP);
+// Long t = document.getLong("t");
+ Long h = document.getLong(HASH);
+ Integer v = document.getInteger(VERSION);
+ String nameSpace = document.getString(NAMESPACE);
+// String uuid = document.getString("uuid");
+// Date wall = document.getDate("wall");
+ Document operation = document.get(OPERATION, Document.class);
+ Document objectID = document.get(OBJECTID, Document.class);
+ event = new ReplicationEvent(operationType, timestamp, v, h, nameSpace, Optional.ofNullable(operation), Optional.ofNullable(objectID), document);
+ } catch (Exception e) {
+ System.out.println(e);
+ }
+
+ return event;
+ }
+
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java b/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
new file mode 100644
index 0000000..54df394
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/OperationType.java
@@ -0,0 +1,28 @@
+package org.apache.connect.mongo.replicator.event;
+
+public enum OperationType {
+
+ INSERT("i"),
+ UPDATE("u"),
+ DELETE("d"),
+ NOOP("n"),
+ DBCOMMAND("c"),
+ CREATED("created"),
+ UNKNOWN("unknown");
+
+ private final String operationStr;
+
+ OperationType(String operationStr) {
+ this.operationStr = operationStr;
+ }
+
+ public static OperationType getOperationType(String operationStr) {
+ for (OperationType operationType : OperationType.values()) {
+ if (operationType.operationStr.equals(operationStr)) {
+ return operationType;
+ }
+ }
+ return UNKNOWN;
+ }
+
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java b/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
new file mode 100644
index 0000000..7719b3e
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
@@ -0,0 +1,153 @@
+package org.apache.connect.mongo.replicator.event;
+
+import io.openmessaging.connector.api.data.EntryType;
+import org.apache.commons.lang3.StringUtils;
+import org.bson.BsonTimestamp;
+import org.bson.Document;
+
+import java.util.Optional;
+
+public class ReplicationEvent {
+
+ private Document document;
+ private OperationType operationType;
+ private Integer v;
+ private Long h;
+ private BsonTimestamp timestamp;
+ private String databaseName;
+ private String collectionName;
+ private String namespace;
+ private Optional<Document> eventData;
+ private Optional<Document> objectId;
+
+
+ public ReplicationEvent() {
+
+ }
+
+
+ public ReplicationEvent(OperationType operationType, BsonTimestamp timestamp, Integer v, Long h, String namespace, Optional<Document> eventData, Optional<Document> objectId, Document document) {
+ this.operationType = operationType;
+ this.v = v;
+ this.h = h;
+ this.timestamp = timestamp;
+ this.namespace = namespace;
+ this.eventData = eventData;
+ this.objectId = objectId;
+ String[] split = StringUtils.split(namespace, ".", 2);
+ this.databaseName = split != null && split.length == 2 ? split[0] : "";
+ this.collectionName = split != null && split.length == 2 ? split[1] : "";
+ this.document = document;
+ }
+
+
+ public OperationType getOperationType() {
+ return operationType;
+ }
+
+ public Integer getV() {
+ return v;
+ }
+
+ public Long getH() {
+ return h;
+ }
+
+ public BsonTimestamp getTimestamp() {
+ return timestamp;
+ }
+
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ public String getCollectionName() {
+ return collectionName;
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public Optional<Document> getEventData() {
+ return eventData;
+ }
+
+ public Optional<Document> getObjectId() {
+ return objectId;
+ }
+
+ public EntryType getEntryType() {
+ switch (operationType) {
+ case UPDATE:
+ return EntryType.UPDATE;
+ case DELETE:
+ return EntryType.DELETE;
+ case INSERT:
+ return EntryType.CREATE;
+ default:
+ return EntryType.CREATE;
+ }
+ }
+
+
+ public void setOperationType(OperationType operationType) {
+ this.operationType = operationType;
+ }
+
+
+ public Document getDocument() {
+ return document;
+ }
+
+ public void setNamespace(String namespace) {
+ this.namespace = namespace;
+ }
+
+ public void setDocument(Document document) {
+ this.document = document;
+ }
+
+ public void setV(Integer v) {
+ this.v = v;
+ }
+
+ public void setH(Long h) {
+ this.h = h;
+ }
+
+ public void setTimestamp(BsonTimestamp timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ public void setDatabaseName(String databaseName) {
+ this.databaseName = databaseName;
+ }
+
+ public void setCollectionName(String collectionName) {
+ this.collectionName = collectionName;
+ }
+
+ public void setEventData(Optional<Document> eventData) {
+ this.eventData = eventData;
+ }
+
+ public void setObjectId(Optional<Document> objectId) {
+ this.objectId = objectId;
+ }
+
+ @Override
+ public String toString() {
+ return "ReplicationEvent{" +
+ "operationType=" + operationType +
+ ", v=" + v +
+ ", h=" + h +
+ ", timestamp=" + timestamp +
+ ", databaseName='" + databaseName + '\'' +
+ ", collectionName='" + collectionName + '\'' +
+ ", namespace='" + namespace + '\'' +
+ ", eventData=" + eventData.toString() +
+ ", objectId=" + objectId.toString() +
+ '}';
+ }
+}
[rocketmq-connect] 07/13: support multiple mongo replicaset
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit f95bf2257d3bb9c57cfb2194876e57af45c5023e
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Tue Aug 13 15:59:38 2019 +0800
support multiple mongo replicaset
---
...ReplicatorConfig.java => SourceTaskConfig.java} | 35 ++---
.../mongo/connector/MongoSourceConnector.java | 4 +-
.../connect/mongo/connector/MongoSourceTask.java | 146 +++++++-------------
.../mongo/connector/builder/MongoDataEntry.java | 113 +++++++++++++++
.../apache/connect/mongo/initsync/InitSync.java | 61 ++++----
.../apache/connect/mongo/replicator/Constants.java | 6 -
.../apache/connect/mongo/replicator/Filter.java | 8 +-
.../connect/mongo/replicator/MongoReplicator.java | 153 ---------------------
.../connect/mongo/replicator/ReplicaSet.java | 101 ++++++++++++++
.../connect/mongo/replicator/ReplicaSetConfig.java | 139 +++++++++++++++++++
.../connect/mongo/replicator/ReplicaSets.java | 70 ++++++++++
.../mongo/replicator/ReplicaSetsContext.java | 129 +++++++++++++++++
.../connect/mongo/replicator/ReplicatorTask.java | 45 +++---
.../replicator/event/DocumentConvertEvent.java | 29 ----
.../mongo/replicator/event/EventConverter.java | 29 ++++
.../mongo/replicator/event/ReplicationEvent.java | 18 ++-
.../java/org/apache/connect/mongo/FilterTest.java | 28 ++--
.../connect/mongo/MongoSourceConnectorTest.java | 69 +++++++++-
.../java/org/apache/connect/mongo/MongoTest.java | 78 +++++++----
.../apache/connect/mongo/ReplicaContextTest.java | 34 +++++
.../org/apache/connect/mongo/ReplicaSetsTest.java | 65 +++++++++
21 files changed, 946 insertions(+), 414 deletions(-)
diff --git a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java b/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
similarity index 83%
rename from src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
rename to src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
index 9097640..b79b2e9 100644
--- a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
+++ b/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
@@ -1,37 +1,37 @@
package org.apache.connect.mongo;
import io.openmessaging.KeyValue;
-import org.apache.commons.lang3.StringUtils;
import org.bson.BsonTimestamp;
import java.lang.reflect.Method;
+import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
-public class MongoReplicatorConfig {
+public class SourceTaskConfig {
private String replicaSet;
private String mongoAddr;
private String mongoUserName;
private String mongoPassWord;
private String interestDbAndCollection;
- private int positionTimeStamp;
- private int positionInc;
- private boolean dataSync;
+ private String positionTimeStamp;
+ private String positionInc;
+ private String dataSync;
private int copyThread = Runtime.getRuntime().availableProcessors();
- public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
+ public static final Set<String> REQUEST_CONFIG = Collections.unmodifiableSet(new HashSet<String>() {
{
add("mongoAddr");
}
- };
+ });
- public int getPositionInc() {
+ public String getPositionInc() {
return positionInc;
}
- public void setPositionInc(int positionInc) {
+ public void setPositionInc(String positionInc) {
this.positionInc = positionInc;
}
@@ -43,11 +43,11 @@ public class MongoReplicatorConfig {
this.copyThread = copyThread;
}
- public int getPositionTimeStamp() {
+ public String getPositionTimeStamp() {
return positionTimeStamp;
}
- public void setPositionTimeStamp(int positionTimeStamp) {
+ public void setPositionTimeStamp(String positionTimeStamp) {
this.positionTimeStamp = positionTimeStamp;
}
@@ -85,11 +85,11 @@ public class MongoReplicatorConfig {
}
- public boolean getDataSync() {
+ public String getDataSync() {
return dataSync;
}
- public void setDataSync(boolean dataSync) {
+ public void setDataSync(String dataSync) {
this.dataSync = dataSync;
}
@@ -148,16 +148,9 @@ public class MongoReplicatorConfig {
}
}
- public String getDataSouce() {
- if (StringUtils.isBlank(replicaSet)) {
- return mongoAddr;
- }
- return replicaSet + ":" + mongoAddr;
- }
-
public BsonTimestamp getPosition() {
- return new BsonTimestamp(positionTimeStamp, positionInc);
+ return new BsonTimestamp(Integer.valueOf(positionTimeStamp), Integer.valueOf(positionInc));
}
}
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
index 9e659c9..2b28ea2 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
@@ -3,7 +3,7 @@ package org.apache.connect.mongo.connector;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.source.SourceConnector;
-import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.SourceTaskConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -18,7 +18,7 @@ public class MongoSourceConnector extends SourceConnector {
@Override
public String verifyAndSetConfig(KeyValue config) {
- for (String requestKey : MongoReplicatorConfig.REQUEST_CONFIG) {
+ for (String requestKey : SourceTaskConfig.REQUEST_CONFIG) {
if (!config.containsKey(requestKey)) {
return "Request config key: " + requestKey;
}
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
index 7272878..407608a 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
@@ -2,91 +2,71 @@ package org.apache.connect.mongo.connector;
import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
-import io.openmessaging.connector.api.data.*;
+import io.openmessaging.connector.api.data.SourceDataEntry;
import io.openmessaging.connector.api.source.SourceTask;
-import org.apache.connect.mongo.MongoReplicatorConfig;
-import org.apache.connect.mongo.replicator.Constants;
-import org.apache.connect.mongo.replicator.MongoReplicator;
-import org.apache.connect.mongo.replicator.event.OperationType;
-import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.SourceTaskConfig;
+import org.apache.connect.mongo.replicator.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
import java.util.Collection;
-import java.util.List;
+import java.util.regex.Pattern;
public class MongoSourceTask extends SourceTask {
private Logger logger = LoggerFactory.getLogger(this.getClass());
- private MongoReplicator mongoReplicator;
+ private SourceTaskConfig sourceTaskConfig;
- private MongoReplicatorConfig replicatorConfig;
+ private ReplicaSets replicaSets;
- private String mongoSource;
+ private ReplicaSetsContext replicaSetsContext;
+
+ private Pattern pattern = Pattern.compile("^[-\\+]?[\\d]*$");
@Override
public Collection<SourceDataEntry> poll() {
- List<SourceDataEntry> res = new ArrayList<>();
- ReplicationEvent event = mongoReplicator.getQueue().poll();
- if (event == null) {
- return new ArrayList<>();
- }
- JSONObject position = position(event);
- Schema schema = new Schema();
- schema.setDataSource(event.getDatabaseName());
- schema.setName(event.getCollectionName());
- schema.setFields(new ArrayList<>());
- buildFieleds(schema);
- DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
- dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(event.getNamespace().replace(".", "-").replace("$", ""))
- .entryType(event.getEntryType());
-
- if (event.getOperationType().ordinal() == OperationType.CREATED.ordinal()) {
- dataEntryBuilder.putFiled(Constants.CREATED, event.getDocument().toJson());
- dataEntryBuilder.putFiled(Constants.NAMESPACE, event.getNamespace());
- } else {
- dataEntryBuilder.putFiled(Constants.OPERATIONTYPE, event.getOperationType().name());
- dataEntryBuilder.putFiled(Constants.TIMESTAMP, event.getTimestamp().getValue());
- dataEntryBuilder.putFiled(Constants.VERSION, event.getV());
- dataEntryBuilder.putFiled(Constants.NAMESPACE, event.getNamespace());
- dataEntryBuilder.putFiled(Constants.PATCH, event.getEventData().isPresent() ? JSONObject.toJSONString(event.getEventData().get()) : "");
- dataEntryBuilder.putFiled(Constants.OBJECTID, event.getObjectId().isPresent() ? JSONObject.toJSONString(event.getObjectId().get()) : "");
- }
- SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap(mongoSource.getBytes(StandardCharsets.UTF_8)),
- ByteBuffer.wrap(position.toJSONString().getBytes(StandardCharsets.UTF_8)));
- res.add(sourceDataEntry);
- return res;
+ return replicaSetsContext.poll();
}
@Override
public void start(KeyValue config) {
try {
- replicatorConfig = new MongoReplicatorConfig();
- replicatorConfig.load(config);
- mongoReplicator = new MongoReplicator(replicatorConfig);
- mongoSource = replicatorConfig.getDataSouce();
- ByteBuffer position = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(
- mongoSource.getBytes()));
-
- if (position != null && position.array().length > 0) {
- String positionJson = new String(position.array(), StandardCharsets.UTF_8);
- JSONObject jsonObject = JSONObject.parseObject(positionJson);
- replicatorConfig.setPositionTimeStamp(jsonObject.getIntValue("timeStamp"));
- replicatorConfig.setPositionInc(jsonObject.getIntValue("inc"));
- replicatorConfig.setDataSync(jsonObject.getBooleanValue(Constants.INITSYNC));
- } else {
- replicatorConfig.setDataSync(true);
- }
- mongoReplicator.start();
- }catch (Throwable throwable) {
- logger.info("task start error", throwable);
+ sourceTaskConfig = new SourceTaskConfig();
+ sourceTaskConfig.load(config);
+ replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
+ replicaSets = ReplicaSets.create(sourceTaskConfig.getMongoAddr());
+ replicaSets.getReplicaConfigByName().forEach((replicaSetName, replicaSetConfig) -> {
+ ByteBuffer byteBuffer = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(
+ replicaSetName.getBytes()));
+ if (byteBuffer != null && byteBuffer.array().length > 0) {
+ String positionJson = new String(byteBuffer.array(), StandardCharsets.UTF_8);
+ ReplicaSetConfig.Position position = JSONObject.parseObject(positionJson, ReplicaSetConfig.Position.class);
+ replicaSetConfig.setPosition(position);
+ } else {
+ ReplicaSetConfig.Position position = replicaSetConfig.emptyPosition();
+ position.setTimeStamp(sourceTaskConfig.getPositionTimeStamp() != null
+ && pattern.matcher(sourceTaskConfig.getPositionTimeStamp()).matches()
+ ? Integer.valueOf(sourceTaskConfig.getPositionTimeStamp()) : 0);
+ position.setInc(sourceTaskConfig.getPositionInc() != null
+ && pattern.matcher(sourceTaskConfig.getPositionInc()).matches()
+ ? Integer.valueOf(sourceTaskConfig.getPositionInc()) : 0);
+ position.setInitSync(StringUtils.equals(sourceTaskConfig.getDataSync(), Constants.INITSYNC) ? true : false);
+ replicaSetConfig.setPosition(position);
+ }
+
+ ReplicaSet replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
+ replicaSetsContext.addReplicaSet(replicaSet);
+ replicaSet.start();
+ });
+
+
+ } catch (Throwable throwable) {
+ logger.error("task start error", throwable);
stop();
}
}
@@ -94,51 +74,19 @@ public class MongoSourceTask extends SourceTask {
@Override
public void stop() {
logger.info("shut down.....");
- mongoReplicator.shutdown();
+ replicaSetsContext.shutdown();
}
@Override
public void pause() {
- mongoReplicator.pause();
+ logger.info("pause replica task...");
+ replicaSetsContext.pause();
}
@Override
public void resume() {
- mongoReplicator.resume();
- }
-
- private void buildFieleds(Schema schema) {
- Field op = new Field(0, Constants.OPERATIONTYPE, FieldType.STRING);
- schema.getFields().add(op);
- Field time = new Field(1, Constants.TIMESTAMP, FieldType.INT64);
- schema.getFields().add(time);
- Field v = new Field(2, Constants.VERSION, FieldType.INT32);
- schema.getFields().add(v);
- Field namespace = new Field(3, Constants.NAMESPACE, FieldType.STRING);
- schema.getFields().add(namespace);
- Field operation = new Field(4, Constants.CREATED, FieldType.STRING);
- schema.getFields().add(operation);
- Field patch = new Field(5, Constants.PATCH, FieldType.STRING);
- schema.getFields().add(patch);
- Field objectId = new Field(6, Constants.OBJECTID, FieldType.STRING);
- schema.getFields().add(objectId);
+ logger.info("resume replica task...");
+ replicaSetsContext.resume();
}
- private JSONObject position(ReplicationEvent event) {
- JSONObject jsonObject = new JSONObject();
- switch (event.getOperationType()) {
- case CREATED:
- jsonObject.put(Constants.POSITION_TIMESTAMP, 0);
- jsonObject.put(Constants.POSITION_INC, 0);
- jsonObject.put(Constants.INITSYNC, true);
- break;
- default:
- jsonObject.put(Constants.POSITION_TIMESTAMP, event.getTimestamp().getTime());
- jsonObject.put(Constants.POSITION_INC, event.getTimestamp().getInc());
- jsonObject.put(Constants.INITSYNC, false);
- break;
- }
- return jsonObject;
-
- }
}
diff --git a/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java b/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
new file mode 100644
index 0000000..7c0db7d
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
@@ -0,0 +1,113 @@
+package org.apache.connect.mongo.connector.builder;
+
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.connector.api.data.*;
+import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.event.OperationType;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.bson.BsonTimestamp;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+
+import static org.apache.connect.mongo.replicator.Constants.*;
+
+public class MongoDataEntry {
+
+ private static String SCHEMA_CREATED_NAME = "mongo_created";
+ private static String SCHEMA_OPLOG_NAME = "mongo_oplog";
+
+ public static SourceDataEntry createSouceDataEntry(ReplicationEvent event, ReplicaSetConfig replicaSetConfig) {
+
+ DataEntryBuilder dataEntryBuilder;
+
+ if (event.getOperationType().equals(OperationType.CREATED)) {
+ Schema schema = createdSchema(replicaSetConfig.getReplicaSetName());
+ dataEntryBuilder = new DataEntryBuilder(schema);
+ dataEntryBuilder.timestamp(System.currentTimeMillis())
+ .queue(event.getNamespace().replace(".", "-").replace("$", "-"))
+ .entryType(event.getEntryType());
+
+ dataEntryBuilder.putFiled(CREATED, event.getDocument().toJson());
+ dataEntryBuilder.putFiled(NAMESPACE, event.getNamespace());
+
+ } else {
+ Schema schema = oplogSchema(replicaSetConfig.getReplicaSetName());
+ dataEntryBuilder = new DataEntryBuilder(schema);
+ dataEntryBuilder.timestamp(System.currentTimeMillis())
+ .queue(event.getNamespace().replace(".", "-").replace("$", "-"))
+ .entryType(event.getEntryType());
+ dataEntryBuilder.putFiled(OPERATIONTYPE, event.getOperationType().name());
+ dataEntryBuilder.putFiled(TIMESTAMP, event.getTimestamp().getValue());
+ dataEntryBuilder.putFiled(VERSION, event.getV());
+ dataEntryBuilder.putFiled(NAMESPACE, event.getNamespace());
+ dataEntryBuilder.putFiled(PATCH, event.getEventData().isPresent() ? JSONObject.toJSONString(event.getEventData().get()) : "");
+ dataEntryBuilder.putFiled(OBJECTID, event.getObjectId().isPresent() ? JSONObject.toJSONString(event.getObjectId().get()) : "");
+ }
+
+
+ String position = createPosition(event, replicaSetConfig);
+ SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
+ ByteBuffer.wrap(replicaSetConfig.getReplicaSetName().getBytes(StandardCharsets.UTF_8)),
+ ByteBuffer.wrap(position.getBytes(StandardCharsets.UTF_8)));
+ return sourceDataEntry;
+ }
+
+
+ private static String createPosition(ReplicationEvent event, ReplicaSetConfig replicaSetConfig) {
+ ReplicaSetConfig.Position position = replicaSetConfig.emptyPosition();
+ BsonTimestamp timestamp = event.getTimestamp();
+ position.setInc(timestamp != null ? timestamp.getInc() : 0);
+ position.setTimeStamp(timestamp != null ? timestamp.getTime() : 0);
+ position.setInitSync(event.getOperationType().equals(OperationType.CREATED) ? true : false);
+ return JSONObject.toJSONString(position);
+
+ }
+
+ private static Schema createdSchema(String dataSourceName) {
+ Schema schema = new Schema();
+ schema.setDataSource(dataSourceName);
+ schema.setName(SCHEMA_CREATED_NAME);
+ schema.setFields(new ArrayList<>());
+ createdField(schema);
+ return schema;
+ }
+
+
+ private static Schema oplogSchema(String dataSourceName) {
+ Schema schema = new Schema();
+ schema.setDataSource(dataSourceName);
+ schema.setName(SCHEMA_OPLOG_NAME);
+ oplogField(schema);
+ return schema;
+ }
+
+
+ private static void createdField(Schema schema) {
+ Field namespace = new Field(0, NAMESPACE, FieldType.STRING);
+ schema.getFields().add(namespace);
+ Field operation = new Field(1, Constants.CREATED, FieldType.STRING);
+ schema.getFields().add(operation);
+ }
+
+ private static void oplogField(Schema schema) {
+ schema.setFields(new ArrayList<>());
+ Field op = new Field(0, OPERATIONTYPE, FieldType.STRING);
+ schema.getFields().add(op);
+ Field time = new Field(1, TIMESTAMP, FieldType.INT64);
+ schema.getFields().add(time);
+ Field v = new Field(2, VERSION, FieldType.INT32);
+ schema.getFields().add(v);
+ Field namespace = new Field(3, NAMESPACE, FieldType.STRING);
+ schema.getFields().add(namespace);
+ Field patch = new Field(4, PATCH, FieldType.STRING);
+ schema.getFields().add(patch);
+ Field objectId = new Field(5, OBJECTID, FieldType.STRING);
+ schema.getFields().add(objectId);
+ }
+
+
+}
diff --git a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
index e12412b..d92e968 100644
--- a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
+++ b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
@@ -3,45 +3,48 @@ package org.apache.connect.mongo.initsync;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoIterable;
-import org.apache.connect.mongo.replicator.event.DocumentConvertEvent;
+import org.apache.connect.mongo.replicator.ReplicaSet;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.ReplicaSetsContext;
+import org.apache.connect.mongo.replicator.event.EventConverter;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
-import org.apache.connect.mongo.replicator.Filter;
-import org.apache.connect.mongo.replicator.MongoReplicator;
-import org.apache.connect.mongo.MongoReplicatorConfig;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashSet;
import java.util.Set;
-import java.util.concurrent.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
public class InitSync {
private Logger logger = LoggerFactory.getLogger(this.getClass());
- private MongoReplicatorConfig mongoReplicatorConfig;
+ private ReplicaSetConfig replicaSetConfig;
private ExecutorService copyExecutor;
private MongoClient mongoClient;
- private Filter filter;
+ private ReplicaSetsContext context;
private int copyThreadCount;
private Set<CollectionMeta> interestCollections;
private CountDownLatch countDownLatch;
- private MongoReplicator mongoReplicator;
+ private ReplicaSet replicaSet;
- public InitSync(MongoReplicatorConfig mongoReplicatorConfig, MongoClient mongoClient, Filter filter, MongoReplicator mongoReplicator) {
- this.mongoReplicatorConfig = mongoReplicatorConfig;
+ public InitSync(ReplicaSetConfig replicaSetConfig, MongoClient mongoClient, ReplicaSetsContext context, ReplicaSet replicaSet) {
+ this.replicaSetConfig = replicaSetConfig;
this.mongoClient = mongoClient;
- this.filter = filter;
- this.mongoReplicator = mongoReplicator;
+ this.context = context;
+ this.replicaSet = replicaSet;
init();
}
public void start() {
for (CollectionMeta collectionMeta : interestCollections) {
- copyExecutor.submit(new CopyRunner(mongoClient, countDownLatch, collectionMeta, mongoReplicator));
+ copyExecutor.submit(new CopyRunner(mongoClient, countDownLatch, collectionMeta, replicaSet));
}
try {
countDownLatch.await();
@@ -53,7 +56,7 @@ public class InitSync {
private void init() {
interestCollections = getInterestCollection();
- copyThreadCount = Math.min(interestCollections.size(), mongoReplicatorConfig.getCopyThread());
+ copyThreadCount = Math.min(interestCollections.size(), context.getCopyThread());
copyExecutor = Executors.newFixedThreadPool(copyThreadCount, new ThreadFactory() {
AtomicInteger threads = new AtomicInteger();
@@ -76,7 +79,7 @@ public class InitSync {
while (collIterator.hasNext()) {
String collectionName = collIterator.next();
CollectionMeta collectionMeta = new CollectionMeta(dataBaseName, collectionName);
- if (filter.filter(collectionMeta)) {
+ if (context.filterMeta(collectionMeta)) {
res.add(collectionMeta);
}
}
@@ -92,40 +95,46 @@ public class InitSync {
private MongoClient mongoClient;
private CountDownLatch countDownLatch;
private CollectionMeta collectionMeta;
- private MongoReplicator mongoReplicator;
+ private ReplicaSet replicaSet;
- public CopyRunner(MongoClient mongoClient, CountDownLatch countDownLatch, CollectionMeta collectionMeta, MongoReplicator mongoReplicator) {
+ public CopyRunner(MongoClient mongoClient, CountDownLatch countDownLatch, CollectionMeta collectionMeta, ReplicaSet replicaSet) {
this.mongoClient = mongoClient;
this.countDownLatch = countDownLatch;
this.collectionMeta = collectionMeta;
- this.mongoReplicator = mongoReplicator;
+ this.replicaSet = replicaSet;
}
@Override
public void run() {
-
+ logger.info("start copy database:{}, collection:{}", collectionMeta.getDatabaseName(), collectionMeta.getCollectionName());
+ int count = 0;
try {
-
MongoCursor<Document> mongoCursor = mongoClient.getDatabase(collectionMeta.getDatabaseName())
.getCollection(collectionMeta.getCollectionName())
.find()
.batchSize(200)
.iterator();
-
- while (mongoReplicator.isRuning() && mongoCursor.hasNext()) {
+ while (replicaSet.isRuning() && mongoCursor.hasNext()) {
+ if (context.initSyncAbort()) {
+ logger.info("init sync database:{}, collection:{} abort, has copy:{} document", collectionMeta.getDatabaseName(), collectionMeta.getCollectionName(), count);
+ return;
+ }
+ count++;
Document document = mongoCursor.next();
- ReplicationEvent event = DocumentConvertEvent.convert(document);
+ ReplicationEvent event = EventConverter.convert(document, replicaSetConfig.getReplicaSetName());
event.setOperationType(OperationType.CREATED);
event.setNamespace(collectionMeta.getNameSpace());
- mongoReplicator.publishEvent(event);
+ context.publishEvent(event, replicaSetConfig);
}
} catch (Exception e) {
- logger.error("init sync database:{}, collection:{} error", collectionMeta.getDatabaseName(), collectionMeta.getNameSpace());
+ context.initSyncError();
+ logger.error("init sync database:{}, collection:{} error", collectionMeta.getDatabaseName(), collectionMeta.getNameSpace(), e);
} finally {
countDownLatch.countDown();
+ replicaSet.shutdown();
}
- logger.info("database:{}, collection:{}, init sync done", collectionMeta.getDatabaseName(), collectionMeta.getCollectionName());
+ logger.info("database:{}, collection:{}, copy {} documents, init sync done", collectionMeta.getDatabaseName(), collectionMeta.getCollectionName(), count);
}
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Constants.java b/src/main/java/org/apache/connect/mongo/replicator/Constants.java
index 668fd91..1a91a57 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/Constants.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/Constants.java
@@ -3,8 +3,6 @@ package org.apache.connect.mongo.replicator;
public class Constants {
- public static final String APPLICATION_NAME = "java-mongo-replicator";
-
public static final String MONGO_LOCAL_DATABASE = "local";
public static final String MONGO_OPLOG_RS = "oplog.rs";
@@ -23,11 +21,7 @@ public class Constants {
public static final String CREATED = "created";
public static final String PATCH = "patch";
- public static final String INITIAL = "initial";
-
- public static final String POSITION_TIMESTAMP = "timeStamp";
- public static final String POSITION_INC = "inc";
public static final String INITSYNC = "initSync";
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Filter.java b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
index 4a62e41..3e431a3 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/Filter.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
@@ -4,7 +4,7 @@ package org.apache.connect.mongo.replicator;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
import org.apache.commons.lang3.StringUtils;
-import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.SourceTaskConfig;
import org.apache.connect.mongo.initsync.CollectionMeta;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
@@ -21,9 +21,9 @@ public class Filter {
private Function<OperationType, Boolean> notNoopFilter;
- public Filter(MongoReplicatorConfig mongoReplicatorConfig) {
+ public Filter(SourceTaskConfig sourceTaskConfig) {
- String interestDbAndCollection = mongoReplicatorConfig.getInterestDbAndCollection();
+ String interestDbAndCollection = sourceTaskConfig.getInterestDbAndCollection();
if (StringUtils.isNotBlank(interestDbAndCollection)) {
JSONObject jsonObject = JSONObject.parseObject(interestDbAndCollection);
@@ -57,7 +57,7 @@ public class Filter {
}
- public boolean filter(CollectionMeta collectionMeta) {
+ public boolean filterMeta(CollectionMeta collectionMeta) {
return dbAndCollectionFilter.apply(collectionMeta);
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java b/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
deleted file mode 100644
index 60b8d3d..0000000
--- a/src/main/java/org/apache/connect/mongo/replicator/MongoReplicator.java
+++ /dev/null
@@ -1,153 +0,0 @@
-package org.apache.connect.mongo.replicator;
-
-import com.mongodb.ConnectionString;
-import com.mongodb.MongoClientSettings;
-import com.mongodb.client.MongoClient;
-import com.mongodb.client.MongoClients;
-import com.mongodb.client.MongoDatabase;
-import com.mongodb.client.MongoIterable;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.connect.mongo.MongoReplicatorConfig;
-import org.apache.connect.mongo.replicator.event.ReplicationEvent;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import static org.apache.connect.mongo.replicator.Constants.*;
-
-
-public class MongoReplicator {
-
- private Logger logger = LoggerFactory.getLogger(this.getClass());
-
- private AtomicBoolean running = new AtomicBoolean();
-
- private MongoReplicatorConfig mongoReplicatorConfig;
-
- private MongoClientSettings clientSettings;
-
- private ConnectionString connectionString;
-
- private MongoClient mongoClient;
-
- private BlockingQueue<ReplicationEvent> queue = new LinkedBlockingQueue<>();
-
- private Filter filter;
-
- private ExecutorService executorService;
-
- private volatile boolean pause = false;
-
- public MongoReplicator(MongoReplicatorConfig mongoReplicatorConfig) {
- this.mongoReplicatorConfig = mongoReplicatorConfig;
- this.filter = new Filter(mongoReplicatorConfig);
- this.executorService = Executors.newSingleThreadExecutor((r) ->new Thread(r, "real_time_replica_thread"));
-
- buildConnectionString();
- }
-
- public void start() {
-
- try {
- if (!running.compareAndSet(false, true)) {
- logger.info("the java mongo replica already start");
- return;
- }
-
- this.clientSettings = MongoClientSettings.builder()
- .applicationName(APPLICATION_NAME)
- .applyConnectionString(connectionString)
- .build();
- this.mongoClient = MongoClients.create(clientSettings);
- this.isReplicaMongo();
- executorService.submit(new ReplicatorTask(this, mongoClient, mongoReplicatorConfig, filter));
- }catch (Exception e) {
- logger.info("start replicator error", e);
- shutdown();
- }
- }
-
-
- private void buildConnectionString() {
- StringBuilder sb = new StringBuilder();
- sb.append("mongodb://");
- if (StringUtils.isNotBlank(mongoReplicatorConfig.getMongoUserName())
- && StringUtils.isNotBlank(mongoReplicatorConfig.getMongoPassWord())) {
- sb.append(mongoReplicatorConfig.getMongoUserName());
- sb.append(":");
- sb.append(mongoReplicatorConfig.getMongoPassWord());
- sb.append("@");
-
- }
- sb.append(mongoReplicatorConfig.getMongoAddr());
- sb.append("/");
- if (StringUtils.isNotBlank(mongoReplicatorConfig.getReplicaSet())) {
- sb.append("?");
- sb.append("replicaSet=");
- sb.append(mongoReplicatorConfig.getReplicaSet());
- }
- this.connectionString = new ConnectionString(sb.toString());
- }
-
-
- public boolean isReplicaMongo() {
- MongoDatabase local = mongoClient.getDatabase(MONGO_LOCAL_DATABASE);
- MongoIterable<String> collectionNames = local.listCollectionNames();
- for (String collectionName : collectionNames) {
- if (MONGO_OPLOG_RS.equals(collectionName)) {
- return true;
- }
- }
- this.shutdown();
- throw new IllegalStateException(String.format("url:%s, set:%s is not replica", mongoReplicatorConfig.getMongoAddr(), mongoReplicatorConfig.getReplicaSet()));
- }
-
- public void shutdown() {
- if (running.compareAndSet(true, false)) {
- if (!this.executorService.isShutdown()) {
- executorService.shutdown();
- }
- if (this.mongoClient != null) {
- this.mongoClient.close();
- }
- }
-
- }
-
- public void publishEvent(ReplicationEvent replicationEvent) {
- while (true) {
- try {
- queue.put(replicationEvent);
- break;
- } catch (Exception e) {
- }
- }
- }
-
-
-
- public void pause() {
- pause = true;
- }
-
- public void resume() {
- pause = false;
- }
-
- public boolean isPause() {
- return pause;
- }
-
- public boolean isRuning() {
- return running.get();
- }
-
- public BlockingQueue<ReplicationEvent> getQueue() {
- return queue;
- }
-}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
new file mode 100644
index 0000000..b141c2b
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
@@ -0,0 +1,101 @@
+package org.apache.connect.mongo.replicator;
+
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.MongoIterable;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.connect.mongo.replicator.Constants.MONGO_LOCAL_DATABASE;
+import static org.apache.connect.mongo.replicator.Constants.MONGO_OPLOG_RS;
+
+
+public class ReplicaSet {
+
+ private Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private AtomicBoolean running = new AtomicBoolean();
+
+ private ReplicaSetConfig replicaSetConfig;
+
+
+ private ReplicaSetsContext replicaSetsContext;
+
+ private MongoClient mongoClient;
+
+ private ExecutorService executorService;
+
+ private volatile boolean pause = false;
+
+ public ReplicaSet(ReplicaSetConfig replicaSetConfig, ReplicaSetsContext replicaSetsContext) {
+ this.replicaSetConfig = replicaSetConfig;
+ this.replicaSetsContext = replicaSetsContext;
+ this.executorService = Executors.newSingleThreadExecutor((r) -> new Thread(r, "real_time_replica_" + replicaSetConfig.getReplicaSetName() + "thread"));
+
+ }
+
+ public void start() {
+
+ try {
+ if (!running.compareAndSet(false, true)) {
+ logger.info("the java mongo replica already start");
+ return;
+ }
+ this.mongoClient = replicaSetsContext.createMongoClient(replicaSetConfig);
+ this.isReplicaMongo();
+ executorService.submit(new ReplicatorTask(this, mongoClient, replicaSetConfig, replicaSetsContext));
+ } catch (Exception e) {
+ logger.error("start replicator:{} error", replicaSetConfig, e);
+ shutdown();
+ }
+ }
+
+
+ public boolean isReplicaMongo() {
+ MongoDatabase local = mongoClient.getDatabase(MONGO_LOCAL_DATABASE);
+ MongoIterable<String> collectionNames = local.listCollectionNames();
+ MongoCursor<String> iterator = collectionNames.iterator();
+ while (iterator.hasNext()) {
+ if (StringUtils.equals(MONGO_OPLOG_RS, iterator.next())) {
+ return true;
+ }
+ }
+ this.shutdown();
+ throw new IllegalStateException(String.format("url:%s, is not replica", replicaSetConfig.getHost()));
+ }
+
+ public void shutdown() {
+ if (running.compareAndSet(true, false)) {
+ if (!this.executorService.isShutdown()) {
+ executorService.shutdown();
+ }
+ if (this.mongoClient != null) {
+ this.mongoClient.close();
+ }
+ }
+
+ }
+
+
+ public void pause() {
+ pause = true;
+ }
+
+ public void resume() {
+ pause = false;
+ }
+
+ public boolean isPause() {
+ return pause;
+ }
+
+ public boolean isRuning() {
+ return running.get();
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
new file mode 100644
index 0000000..4b8d148
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
@@ -0,0 +1,139 @@
+package org.apache.connect.mongo.replicator;
+
+import org.bson.BsonTimestamp;
+
+import java.util.Objects;
+
+public class ReplicaSetConfig {
+
+
+ private String shardName;
+ private String replicaSetName;
+ private String host;
+ private Position position;
+
+
+ public Position getPosition() {
+ return position;
+ }
+
+ public void setPosition(Position position) {
+ this.position = position;
+ }
+
+ public String getShardName() {
+ return shardName;
+ }
+
+ public void setShardName(String shardName) {
+ this.shardName = shardName;
+ }
+
+ public String getReplicaSetName() {
+ return replicaSetName;
+ }
+
+ public void setReplicaSetName(String replicaSetName) {
+ this.replicaSetName = replicaSetName;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public void setHost(String host) {
+ this.host = host;
+ }
+
+ public ReplicaSetConfig(String shardName, String replicaSetName, String host) {
+ this.shardName = shardName;
+ this.replicaSetName = replicaSetName;
+ this.host = host;
+ }
+
+ public Position emptyPosition() {
+ return new Position(0, 0, true);
+ }
+
+
+ public class Position {
+ private int timeStamp;
+ private int inc;
+ private boolean initSync;
+
+
+ public int getTimeStamp() {
+ return timeStamp;
+ }
+
+ public void setTimeStamp(int timeStamp) {
+ this.timeStamp = timeStamp;
+ }
+
+ public int getInc() {
+ return inc;
+ }
+
+ public void setInc(int inc) {
+ this.inc = inc;
+ }
+
+ public boolean isInitSync() {
+ return initSync;
+ }
+
+ public void setInitSync(boolean initSync) {
+ this.initSync = initSync;
+ }
+
+
+ public Position(int timeStamp, int inc, boolean initSync) {
+ this.timeStamp = timeStamp;
+ this.inc = inc;
+ this.initSync = initSync;
+ }
+
+ public boolean isValid() {
+ return timeStamp > 0;
+ }
+
+ public BsonTimestamp converBsonTimeStamp() {
+ return new BsonTimestamp(timeStamp, inc);
+ }
+
+ @Override
+ public String toString() {
+ return "Position{" +
+ "timeStamp=" + timeStamp +
+ ", inc=" + inc +
+ ", initSync=" + initSync +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Position position = (Position) o;
+ return timeStamp == position.timeStamp &&
+ inc == position.inc &&
+ initSync == position.initSync;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(timeStamp, inc, initSync);
+ }
+ }
+
+
+ @Override
+ public String toString() {
+ return "ReplicaSetConfig{" +
+ "shardName='" + shardName + '\'' +
+ ", replicaSetName='" + replicaSetName + '\'' +
+ ", host='" + host + '\'' +
+ ", position=" + position +
+ '}';
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java
new file mode 100644
index 0000000..9184b90
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java
@@ -0,0 +1,70 @@
+package org.apache.connect.mongo.replicator;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ReplicaSets {
+
+ private static final Pattern HOST_PATTERN = Pattern.compile("((([^=]+)[=])?(([^/]+)\\/))?(.+)");
+
+
+ private final Map<String, ReplicaSetConfig> replicaConfigByName = new HashMap<>();
+
+
+ public ReplicaSets(Set<ReplicaSetConfig> replicaSetConfigs) {
+ replicaSetConfigs.forEach(replicaSetConfig -> {
+ if (StringUtils.isNotBlank(replicaSetConfig.getReplicaSetName())) {
+ replicaConfigByName.put(replicaSetConfig.getReplicaSetName(), replicaSetConfig);
+ }
+ });
+
+ validate();
+ }
+
+ public static ReplicaSets create(String hosts) {
+ Set<ReplicaSetConfig> replicaSetConfigs = new HashSet<>();
+ if (hosts != null) {
+ for (String replicaSetStr : StringUtils.split(hosts.trim(), ";")) {
+ if (StringUtils.isNotBlank(replicaSetStr)) {
+ ReplicaSetConfig replicaSetConfig = parseReplicaSetStr(replicaSetStr);
+ if (replicaSetConfig != null) {
+ replicaSetConfigs.add(replicaSetConfig);
+ }
+ }
+ }
+ }
+ return new ReplicaSets(replicaSetConfigs);
+ }
+
+
+ private static ReplicaSetConfig parseReplicaSetStr(String hosts) {
+ if (hosts != null) {
+ Matcher matcher = HOST_PATTERN.matcher(hosts);
+ if (matcher.matches()) {
+ String shard = matcher.group(3);
+ String replicaSetName = matcher.group(5);
+ String host = matcher.group(6);
+ if (host != null && host.trim().length() != 0) {
+ return new ReplicaSetConfig(shard, replicaSetName, host);
+ }
+ }
+ }
+ return null;
+ }
+
+ private void validate() {
+ Validate.isTrue(replicaConfigByName.size() > 0, "task config mongoAdd need special replicaSet addr");
+
+ }
+
+ public Map<String, ReplicaSetConfig> getReplicaConfigByName() {
+ return replicaConfigByName;
+ }
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
new file mode 100644
index 0000000..f66ca14
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
@@ -0,0 +1,129 @@
+package org.apache.connect.mongo.replicator;
+
+import com.mongodb.ConnectionString;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import io.openmessaging.connector.api.data.SourceDataEntry;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.SourceTaskConfig;
+import org.apache.connect.mongo.connector.builder.MongoDataEntry;
+import org.apache.connect.mongo.initsync.CollectionMeta;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ReplicaSetsContext {
+
+ private BlockingQueue<SourceDataEntry> dataEntryQueue;
+
+ private SourceTaskConfig taskConfig;
+
+ private List<ReplicaSet> replicaSets;
+
+ private AtomicBoolean initSyncAbort = new AtomicBoolean();
+
+ private Filter filter;
+
+ public ReplicaSetsContext(SourceTaskConfig taskConfig) {
+ this.taskConfig = taskConfig;
+ this.replicaSets = new CopyOnWriteArrayList<>();
+ this.dataEntryQueue = new LinkedBlockingDeque<>();
+ this.filter = new Filter(taskConfig);
+ }
+
+
+ public MongoClient createMongoClient(ReplicaSetConfig replicaSetConfig) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("mongodb://");
+ if (StringUtils.isNotBlank(taskConfig.getMongoUserName())
+ && StringUtils.isNotBlank(taskConfig.getMongoPassWord())) {
+ sb.append(taskConfig.getMongoUserName());
+ sb.append(":");
+ sb.append(taskConfig.getMongoPassWord());
+ sb.append("@");
+
+ }
+ sb.append(replicaSetConfig.getHost());
+ sb.append("/");
+ if (StringUtils.isNotBlank(replicaSetConfig.getReplicaSetName())) {
+ sb.append("?");
+ sb.append("replicaSet=");
+ sb.append(replicaSetConfig.getReplicaSetName());
+ }
+ ConnectionString connectionString = new ConnectionString(sb.toString());
+ return MongoClients.create(connectionString);
+ }
+
+
+ public boolean filterEvent(ReplicationEvent event) {
+ return filter.filterEvent(event);
+ }
+
+
+ public boolean filterMeta(CollectionMeta collectionMeta) {
+ return filter.filterMeta(collectionMeta);
+ }
+
+
+ public int getCopyThread() {
+ return taskConfig.getCopyThread() > 0 ? taskConfig.getCopyThread() : Runtime.getRuntime().availableProcessors();
+ }
+
+
+ public void addReplicaSet(ReplicaSet replicaSet) {
+ this.replicaSets.add(replicaSet);
+ }
+
+
+ public void shutdown() {
+ replicaSets.forEach(ReplicaSet::shutdown);
+ }
+
+ public void pause() {
+ replicaSets.forEach(ReplicaSet::pause);
+ }
+
+
+ public void resume() {
+ replicaSets.forEach(ReplicaSet::resume);
+ }
+
+
+ public void publishEvent(ReplicationEvent event, ReplicaSetConfig replicaSetConfig) {
+ SourceDataEntry sourceDataEntry = MongoDataEntry.createSouceDataEntry(event, replicaSetConfig);
+ while (true) {
+ try {
+ dataEntryQueue.put(sourceDataEntry);
+ break;
+ } catch (InterruptedException e) {
+ }
+ }
+
+ }
+
+ public Collection<SourceDataEntry> poll() {
+ List<SourceDataEntry> res = new ArrayList<>();
+ if (dataEntryQueue.drainTo(res, 20) == 0) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ }
+ }
+ return res;
+ }
+
+ public boolean initSyncAbort() {
+ return initSyncAbort.get();
+ }
+
+ public void initSyncError() {
+ initSyncAbort.set(true);
+ }
+
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
index 766225f..bf4ebac 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
@@ -6,9 +6,8 @@ import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
-import org.apache.connect.mongo.MongoReplicatorConfig;
import org.apache.connect.mongo.initsync.InitSync;
-import org.apache.connect.mongo.replicator.event.DocumentConvertEvent;
+import org.apache.connect.mongo.replicator.event.EventConverter;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
import org.bson.Document;
import org.slf4j.Logger;
@@ -19,34 +18,34 @@ public class ReplicatorTask implements Runnable {
private Logger logger = LoggerFactory.getLogger(this.getClass());
- private MongoReplicator mongoReplicator;
+ private ReplicaSet replicaSet;
private MongoClient mongoClient;
- private MongoReplicatorConfig mongoReplicatorConfig;
+ private ReplicaSetConfig replicaSetConfig;
- private Filter filter;
+ private ReplicaSetsContext replicaSetsContext;
- public ReplicatorTask(MongoReplicator mongoReplicator, MongoClient mongoClient, MongoReplicatorConfig mongoReplicatorConfig, Filter filter) {
- this.mongoReplicator = mongoReplicator;
- this.mongoReplicatorConfig = mongoReplicatorConfig;
+ public ReplicatorTask(ReplicaSet replicaSet, MongoClient mongoClient, ReplicaSetConfig replicaSetConfig, ReplicaSetsContext replicaSetsContext) {
+ this.replicaSet = replicaSet;
+ this.replicaSetConfig = replicaSetConfig;
this.mongoClient = mongoClient;
- this.filter = filter;
+ this.replicaSetsContext = replicaSetsContext;
}
@Override
public void run() {
- if (mongoReplicatorConfig.getDataSync()) {
- InitSync initSync = new InitSync(mongoReplicatorConfig, mongoClient, filter, mongoReplicator);
+ if (replicaSetConfig.getPosition() == null || replicaSetConfig.getPosition().isInitSync()) {
+ InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
initSync.start();
}
MongoDatabase localDataBase = mongoClient.getDatabase(Constants.MONGO_LOCAL_DATABASE);
FindIterable<Document> iterable;
- if (mongoReplicatorConfig.getPositionTimeStamp() > 0 && mongoReplicatorConfig.getPositionTimeStamp() < System.currentTimeMillis()) {
+ if (replicaSetConfig.getPosition().isValid()) {
iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find(
- Filters.gt("ts", mongoReplicatorConfig.getPosition()));
+ Filters.gt("ts", replicaSetConfig.getPosition().converBsonTimeStamp()));
} else {
iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
}
@@ -56,26 +55,28 @@ public class ReplicatorTask implements Runnable {
.batchSize(200)
.iterator();
- while (mongoReplicator.isRuning()) {
+ while (replicaSet.isRuning()) {
try {
executorCursor(cursor);
- Thread.sleep(100);
} catch (Exception e) {
- e.printStackTrace();
+ logger.error("replicaSet:{} shutdown.....", replicaSetConfig, e);
} finally {
- logger.error("mongoReplicator shutdown.....");
- mongoReplicator.shutdown();
+ if (cursor != null) {
+ cursor.close();
+ }
+ replicaSet.shutdown();
}
}
+ logger.info("replicaSet:{}, already shutdown, replicaTask end of life cycle", replicaSetConfig);
}
private void executorCursor(MongoCursor<Document> cursor) {
- while (cursor.hasNext() && !mongoReplicator.isPause()) {
+ while (cursor.hasNext() && !replicaSet.isPause()) {
Document document = cursor.next();
- ReplicationEvent event = DocumentConvertEvent.convert(document);
- if (filter.filterEvent(event)) {
- mongoReplicator.publishEvent(event);
+ ReplicationEvent event = EventConverter.convert(document, replicaSetConfig.getReplicaSetName());
+ if (replicaSetsContext.filterEvent(event)) {
+ replicaSetsContext.publishEvent(event, replicaSetConfig);
}
}
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java b/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java
deleted file mode 100644
index a18aa52..0000000
--- a/src/main/java/org/apache/connect/mongo/replicator/event/DocumentConvertEvent.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.apache.connect.mongo.replicator.event;
-
-import org.bson.BsonTimestamp;
-import org.bson.Document;
-
-import java.util.Optional;
-
-import static org.apache.connect.mongo.replicator.Constants.*;
-
-
-public class DocumentConvertEvent {
-
-
- public static ReplicationEvent convert(Document document) {
-
- OperationType operationType = OperationType.getOperationType(document.getString(OPERATIONTYPE));
- BsonTimestamp timestamp = (BsonTimestamp) document.get(TIMESTAMP);
-// Long t = document.getLong("t");
- Long h = document.getLong(HASH);
- Integer v = document.getInteger(VERSION);
- String nameSpace = document.getString(NAMESPACE);
-// String uuid = document.getString("uuid");
-// Date wall = document.getDate("wall");
- Document operation = document.get(OPERATION, Document.class);
- Document objectID = document.get(OBJECTID, Document.class);
- return new ReplicationEvent(operationType, timestamp, v, h, nameSpace, Optional.ofNullable(operation), Optional.ofNullable(objectID), document);
- }
-
-}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java b/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java
new file mode 100644
index 0000000..57b4ac8
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java
@@ -0,0 +1,29 @@
+package org.apache.connect.mongo.replicator.event;
+
+import org.bson.BsonTimestamp;
+import org.bson.Document;
+
+import java.util.Optional;
+
+import static org.apache.connect.mongo.replicator.Constants.*;
+
+
+public class EventConverter {
+
+
+ public static ReplicationEvent convert(Document document, String replicaSetName) {
+
+ ReplicationEvent event = new ReplicationEvent();
+ event.setOperationType(OperationType.getOperationType(document.getString(OPERATIONTYPE)));
+ event.setTimestamp(document.get(TIMESTAMP, BsonTimestamp.class));
+ event.setH(document.getLong(HASH));
+ event.setV(document.getInteger(VERSION));
+ event.setNamespace(document.getString(NAMESPACE));
+ event.setEventData(Optional.ofNullable(document.get(OPERATION, Document.class)));
+ event.setObjectId(Optional.ofNullable(document.get(OBJECTID, Document.class)));
+ event.setReplicaSetName(replicaSetName);
+ event.setDocument(document);
+ return event;
+ }
+
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java b/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
index 7719b3e..283e9d6 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
@@ -19,6 +19,7 @@ public class ReplicationEvent {
private String namespace;
private Optional<Document> eventData;
private Optional<Document> objectId;
+ private String replicaSetName;
public ReplicationEvent() {
@@ -136,18 +137,29 @@ public class ReplicationEvent {
this.objectId = objectId;
}
+
+ public void setReplicaSetName(String replicaSetName) {
+ this.replicaSetName = replicaSetName;
+ }
+
+ public String getReplicaSetName() {
+ return replicaSetName;
+ }
+
@Override
public String toString() {
return "ReplicationEvent{" +
- "operationType=" + operationType +
+ "document=" + document +
+ ", operationType=" + operationType +
", v=" + v +
", h=" + h +
", timestamp=" + timestamp +
", databaseName='" + databaseName + '\'' +
", collectionName='" + collectionName + '\'' +
", namespace='" + namespace + '\'' +
- ", eventData=" + eventData.toString() +
- ", objectId=" + objectId.toString() +
+ ", eventData=" + eventData +
+ ", objectId=" + objectId +
+ ", replicaSetName='" + replicaSetName + '\'' +
'}';
}
}
diff --git a/src/test/java/org/apache/connect/mongo/FilterTest.java b/src/test/java/org/apache/connect/mongo/FilterTest.java
index 2b14b13..60e2514 100644
--- a/src/test/java/org/apache/connect/mongo/FilterTest.java
+++ b/src/test/java/org/apache/connect/mongo/FilterTest.java
@@ -17,12 +17,12 @@ import java.util.Map;
public class FilterTest {
- private MongoReplicatorConfig config;
+ private SourceTaskConfig sourceTaskConfig;
private Map<String, List<String>> insterest;
@Before
public void init() {
- config = new MongoReplicatorConfig();
+ sourceTaskConfig = new SourceTaskConfig();
insterest = new HashMap<>();
}
@@ -31,18 +31,18 @@ public class FilterTest {
List<String> collections = new ArrayList<>();
collections.add("person");
insterest.put("test", collections);
- config.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
- Filter filter = new Filter(config);
- Assert.assertTrue(filter.filter(new CollectionMeta("test", "person")));
- Assert.assertFalse(filter.filter(new CollectionMeta("test", "person01")));
+ sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
+ Filter filter = new Filter(sourceTaskConfig);
+ Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "person")));
+ Assert.assertFalse(filter.filterMeta(new CollectionMeta("test", "person01")));
}
@Test
public void testBlankDb() {
- Filter filter = new Filter(config);
- Assert.assertTrue(filter.filter(new CollectionMeta("test" ,"test")));
- Assert.assertTrue(filter.filter(new CollectionMeta("test1" ,"test01")));
+ Filter filter = new Filter(sourceTaskConfig);
+ Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "test")));
+ Assert.assertTrue(filter.filterMeta(new CollectionMeta("test1", "test01")));
}
@@ -51,17 +51,17 @@ public class FilterTest {
List<String> collections = new ArrayList<>();
collections.add("*");
insterest.put("test", collections);
- config.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
- Filter filter = new Filter(config);
- Assert.assertTrue(filter.filter(new CollectionMeta("test", "testsad")));
- Assert.assertTrue(filter.filter(new CollectionMeta("test", "tests032")));
+ sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
+ Filter filter = new Filter(sourceTaskConfig);
+ Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "testsad")));
+ Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "tests032")));
}
@Test
public void testFilterEvent() {
- Filter filter = new Filter(config);
+ Filter filter = new Filter(sourceTaskConfig);
ReplicationEvent replicationEvent = new ReplicationEvent();
replicationEvent.setOperationType(OperationType.NOOP);
Assert.assertFalse(filter.filterEvent(replicationEvent));
diff --git a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
index 04e1374..d921e63 100644
--- a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
@@ -1,22 +1,40 @@
package org.apache.connect.mongo;
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.connector.api.data.EntryType;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.SourceDataEntry;
import io.openmessaging.internal.DefaultKeyValue;
-import org.apache.commons.lang3.StringUtils;
import org.apache.connect.mongo.connector.MongoSourceConnector;
import org.apache.connect.mongo.connector.MongoSourceTask;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.ReplicaSetsContext;
+import org.apache.connect.mongo.replicator.event.OperationType;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
+import org.bson.BsonTimestamp;
+import org.bson.Document;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.LinkedBlockingQueue;
+
public class MongoSourceConnectorTest {
private MongoSourceConnector mongoSourceConnector;
private DefaultKeyValue keyValue;
+ private SourceTaskConfig sourceTaskConfig;
@Before
public void before() {
mongoSourceConnector = new MongoSourceConnector();
keyValue = new DefaultKeyValue();
+ sourceTaskConfig = new SourceTaskConfig();
+
}
@Test
@@ -27,19 +45,58 @@ public class MongoSourceConnectorTest {
@Test
public void verifyConfig() {
- keyValue.put("mongoAddr", "127.0.0.1");
+ keyValue.put("mongoAddr", "shardName=replicaName:127.0.0.1:27017");
String s = mongoSourceConnector.verifyAndSetConfig(keyValue);
- Assert.assertTrue(s.contains("Request config key:"));
- keyValue.put("mongoPort", 27017);
- s = mongoSourceConnector.verifyAndSetConfig(keyValue);
- Assert.assertTrue(StringUtils.isBlank(s));
+ Assert.assertTrue(s.contains("Request sourceTaskConfig key:"));
}
+ @Test
+ public void testPoll() throws Exception {
+ LinkedBlockingQueue<SourceDataEntry> entries = new LinkedBlockingQueue<>();
+ ReplicaSetsContext context = new ReplicaSetsContext(sourceTaskConfig);
+ Field dataEntryQueue = ReplicaSetsContext.class.getDeclaredField("dataEntryQueue");
+ dataEntryQueue.setAccessible(true);
+ dataEntryQueue.set(context, entries);
+ ReplicationEvent event = new ReplicationEvent();
+ event.setOperationType(OperationType.INSERT);
+ event.setNamespace("test.person");
+ event.setTimestamp(new BsonTimestamp(1565609506, 1));
+ event.setDocument(new Document("testKey", "testValue"));
+ event.setH(324243242L);
+ event.setEventData(Optional.ofNullable(new Document("testEventKey", "testEventValue")));
+ event.setObjectId(Optional.empty());
+ context.publishEvent(event, new ReplicaSetConfig("", "testReplicaName", "localhost:27017"));
+ List<SourceDataEntry> sourceDataEntries = (List<SourceDataEntry>) context.poll();
+ Assert.assertTrue(sourceDataEntries.size() == 1);
+
+ SourceDataEntry sourceDataEntry = sourceDataEntries.get(0);
+ Assert.assertEquals("test-person", sourceDataEntry.getQueueName());
+
+ ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
+ Assert.assertEquals("testReplicaName", new String(sourcePartition.array()));
+ ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
+ ReplicaSetConfig.Position position = JSONObject.parseObject(new String(sourcePosition.array()), ReplicaSetConfig.Position.class);
+ Assert.assertEquals(position.getTimeStamp(), 1565609506);
+ Assert.assertEquals(position.getInc(), 1);
+ Assert.assertEquals(position.isInitSync(), false);
+ EntryType entryType = sourceDataEntry.getEntryType();
+ Assert.assertEquals(EntryType.CREATE, entryType);
+
+ String queueName = sourceDataEntry.getQueueName();
+ Assert.assertEquals("test-person", queueName);
+
+ Schema schema = sourceDataEntry.getSchema();
+ Assert.assertTrue(schema.getFields().size() == 6);
+ Object[] payload = sourceDataEntry.getPayload();
+ Assert.assertTrue(payload.length == 6);
+
+ }
+
}
diff --git a/src/test/java/org/apache/connect/mongo/MongoTest.java b/src/test/java/org/apache/connect/mongo/MongoTest.java
index cc83fbe..7b0291a 100644
--- a/src/test/java/org/apache/connect/mongo/MongoTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoTest.java
@@ -7,11 +7,14 @@ import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import io.openmessaging.connector.api.data.EntryType;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.SourceDataEntry;
import org.apache.connect.mongo.initsync.InitSync;
import org.apache.connect.mongo.replicator.Constants;
-import org.apache.connect.mongo.replicator.Filter;
-import org.apache.connect.mongo.replicator.MongoReplicator;
-import org.apache.connect.mongo.replicator.event.DocumentConvertEvent;
+import org.apache.connect.mongo.replicator.ReplicaSet;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.ReplicaSetsContext;
+import org.apache.connect.mongo.replicator.event.EventConverter;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
import org.bson.BsonTimestamp;
@@ -21,12 +24,8 @@ import org.junit.Before;
import org.junit.Test;
import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.TimeUnit;
+import java.nio.ByteBuffer;
+import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
public class MongoTest {
@@ -36,7 +35,7 @@ public class MongoTest {
@Before
public void before() {
MongoClientSettings.Builder builder = MongoClientSettings.builder();
- builder.applyConnectionString(new ConnectionString("mongodb://127.0.0.1:27018"));
+ builder.applyConnectionString(new ConnectionString("mongodb://127.0.0.1:27077"));
mongoClient = MongoClients.create(builder.build());
}
@@ -52,13 +51,14 @@ public class MongoTest {
Document document = new Document();
document.put("test", "test");
oplog.put(Constants.OPERATION, document);
- ReplicationEvent event = DocumentConvertEvent.convert(oplog);
+ ReplicationEvent event = EventConverter.convert(oplog, "testR");
Assert.assertEquals(timestamp, event.getTimestamp());
Assert.assertEquals("test.person", event.getNamespace());
Assert.assertTrue(11111L == event.getH());
Assert.assertEquals(OperationType.INSERT, event.getOperationType());
Assert.assertEquals(EntryType.CREATE, event.getEntryType());
Assert.assertEquals(document, event.getEventData().get());
+ Assert.assertEquals("testR", event.getReplicaSetName());
}
@@ -68,38 +68,58 @@ public class MongoTest {
public void testInitSyncCopy() throws NoSuchFieldException, IllegalAccessException, InterruptedException {
MongoCollection<Document> collection = mongoClient.getDatabase("test").getCollection("person");
collection.deleteMany(new Document());
- int count = 1;
- List<Document> documents = new ArrayList<>(count);
+ int count = 1000;
+ List<String> documents = new ArrayList<>(count);
for (int i = 0; i < count; i++) {
Document document = new Document();
document.put("name", "test" + i);
document.put("age", i);
document.put("sex", i % 2 == 0 ? "boy" : "girl");
collection.insertOne(document);
- documents.add(document);
+ documents.add(document.getObjectId("_id").toHexString());
}
- MongoReplicatorConfig config = new MongoReplicatorConfig();
+ SourceTaskConfig sourceTaskConfig = new SourceTaskConfig();
Map<String, List<String>> insterest = new HashMap<>();
List<String> collections = new ArrayList<>();
collections.add("*");
insterest.put("test", collections);
- config.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
- MongoReplicator mongoReplicator = new MongoReplicator(config);
- Field running = MongoReplicator.class.getDeclaredField("running");
+ sourceTaskConfig.setInterestDbAndCollection(JSONObject.toJSONString(insterest));
+ ReplicaSetConfig replicaSetConfig = new ReplicaSetConfig("", "test", "localhost");
+ ReplicaSetsContext replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
+ ReplicaSet replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
+ Field running = ReplicaSet.class.getDeclaredField("running");
running.setAccessible(true);
- running.set(mongoReplicator, new AtomicBoolean(true));
- InitSync initSync = new InitSync(config, mongoClient, new Filter(config), mongoReplicator);
+ running.set(replicaSet, new AtomicBoolean(true));
+ InitSync initSync = new InitSync(replicaSetConfig, mongoClient, replicaSetsContext, replicaSet);
initSync.start();
- BlockingQueue<ReplicationEvent> queue = mongoReplicator.getQueue();
- while (count > 0) {
- count--;
- ReplicationEvent event = queue.poll(100, TimeUnit.MILLISECONDS);
- Assert.assertTrue(event.getOperationType().equals(OperationType.CREATED));
- Assert.assertNotNull(event.getDocument());
- Assert.assertTrue(documents.contains(event.getDocument()));
- }
-
+ int syncCount = 0;
+ while (syncCount < count) {
+ Collection<SourceDataEntry> sourceDataEntries = replicaSetsContext.poll();
+ Assert.assertNotNull(sourceDataEntries);
+ for (SourceDataEntry sourceDataEntry : sourceDataEntries) {
+ ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
+ Assert.assertEquals("test", new String(sourcePartition.array()));
+ ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
+ ReplicaSetConfig.Position position = replicaSetConfig.emptyPosition();
+ position.setInitSync(true);
+ position.setTimeStamp(0);
+ position.setInc(0);
+ Assert.assertEquals(position, JSONObject.parseObject(new String(sourcePosition.array()), ReplicaSetConfig.Position.class));
+ EntryType entryType = sourceDataEntry.getEntryType();
+ Assert.assertEquals(EntryType.CREATE, entryType);
+ String queueName = sourceDataEntry.getQueueName();
+ Assert.assertEquals("test-person", queueName);
+ Schema schema = sourceDataEntry.getSchema();
+ Assert.assertTrue(schema.getFields().size() == 2);
+ Object[] payload = sourceDataEntry.getPayload();
+ Assert.assertTrue(payload.length == 2);
+ Assert.assertEquals(payload[0].toString(), "test.person");
+ Assert.assertTrue(documents.contains(JSONObject.parseObject(payload[1].toString(), Document.class).get("_id", JSONObject.class).getString("$oid")));
+ syncCount++;
+ }
+ }
+ Assert.assertTrue(syncCount == count);
}
}
diff --git a/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java b/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
new file mode 100644
index 0000000..8613c42
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
@@ -0,0 +1,34 @@
+package org.apache.connect.mongo;
+
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.MongoIterable;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.ReplicaSetsContext;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ReplicaContextTest {
+
+ private ReplicaSetsContext context;
+
+ @Before
+ public void before() {
+ SourceTaskConfig sourceTaskConfig = new SourceTaskConfig();
+ context = new ReplicaSetsContext(sourceTaskConfig);
+ }
+
+
+ @Test
+ public void testCreateMongoClient() {
+ MongoClient mongoClient = context.createMongoClient(new ReplicaSetConfig("shardName1", "rep1", "127.0.0.1:27017"));
+ MongoIterable<String> collectionNames = mongoClient.getDatabase("local").listCollectionNames();
+ MongoCursor<String> iterator = collectionNames.iterator();
+ while (iterator.hasNext()) {
+ Assert.assertTrue(StringUtils.isNoneBlank(iterator.next()));
+ }
+ }
+
+}
diff --git a/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java b/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java
new file mode 100644
index 0000000..e69eac6
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java
@@ -0,0 +1,65 @@
+package org.apache.connect.mongo;
+
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.ReplicaSets;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class ReplicaSetsTest {
+
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCreatReplicaSetsException01() {
+ ReplicaSets.create("");
+ }
+
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCreatReplicaSetsException02() {
+ ReplicaSets.create("127.0.0.1:27081");
+ }
+
+
+ @Test
+ public void testCreatReplicaSets01() {
+ ReplicaSets replicaSets = ReplicaSets.create("replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083");
+ Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
+ Assert.assertTrue(replicaSetConfigMap.size() == 1);
+ Assert.assertNotNull(replicaSetConfigMap.get("replicaName1"));
+ Assert.assertEquals("127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083", replicaSetConfigMap.get("replicaName1").getHost());
+ Assert.assertEquals("replicaName1", replicaSetConfigMap.get("replicaName1").getReplicaSetName());
+ }
+
+
+ @Test
+ public void testCreatReplicaSets02() {
+ ReplicaSets replicaSets = ReplicaSets.create("shardName1=replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083");
+ Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
+ Assert.assertTrue(replicaSetConfigMap.size() == 1);
+ Assert.assertNotNull(replicaSetConfigMap.get("replicaName1"));
+ Assert.assertEquals("127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083", replicaSetConfigMap.get("replicaName1").getHost());
+ Assert.assertEquals("replicaName1", replicaSetConfigMap.get("replicaName1").getReplicaSetName());
+ Assert.assertEquals("shardName1", replicaSetConfigMap.get("replicaName1").getShardName());
+ }
+
+
+ @Test
+ public void testCreatReplicaSets03() {
+ ReplicaSets replicaSets = ReplicaSets.create("shardName1=replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083;shardName2=replicaName2/127.0.0.1:27281,127.0.0.1:27282,127.0.0.1:27283");
+ Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
+ Assert.assertTrue(replicaSetConfigMap.size() == 2);
+ Assert.assertNotNull(replicaSetConfigMap.get("replicaName1"));
+ Assert.assertEquals("127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083", replicaSetConfigMap.get("replicaName1").getHost());
+ Assert.assertEquals("replicaName1", replicaSetConfigMap.get("replicaName1").getReplicaSetName());
+ Assert.assertEquals("shardName1", replicaSetConfigMap.get("replicaName1").getShardName());
+
+
+ Assert.assertNotNull(replicaSetConfigMap.get("replicaName2"));
+ Assert.assertEquals("127.0.0.1:27281,127.0.0.1:27282,127.0.0.1:27283", replicaSetConfigMap.get("replicaName2").getHost());
+ Assert.assertEquals("replicaName2", replicaSetConfigMap.get("replicaName2").getReplicaSetName());
+ Assert.assertEquals("shardName2", replicaSetConfigMap.get("replicaName2").getShardName());
+ }
+
+}
[rocketmq-connect] 09/13: reformat code and add more test case
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit f193fcdc1c654837d9dfee60ea4b2e56afbd6d5d
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Thu Aug 15 15:22:58 2019 +0800
reformat code and add more test case
---
README.md | 12 ++
.../org/apache/connect/mongo/SourceTaskConfig.java | 116 +++++++++++++-
.../mongo/connector/MongoSourceConnector.java | 6 +-
.../connect/mongo/connector/MongoSourceTask.java | 26 ++--
.../mongo/connector/builder/MongoDataEntry.java | 39 ++---
.../connect/mongo/initsync/CollectionMeta.java | 6 +-
.../apache/connect/mongo/initsync/InitSync.java | 30 ++--
.../apache/connect/mongo/replicator/Constants.java | 6 -
.../apache/connect/mongo/replicator/Filter.java | 15 +-
.../mongo/replicator/MongoClientFactory.java | 113 ++++++++++++++
.../connect/mongo/replicator/ReplicaSet.java | 11 +-
.../connect/mongo/replicator/ReplicaSetConfig.java | 45 +++---
.../connect/mongo/replicator/ReplicaSets.java | 8 +-
.../mongo/replicator/ReplicaSetsContext.java | 43 +----
.../connect/mongo/replicator/ReplicatorTask.java | 16 +-
.../mongo/replicator/event/EventConverter.java | 13 +-
.../mongo/replicator/event/ReplicationEvent.java | 36 ++---
.../java/org/apache/connect/mongo/FilterTest.java | 15 +-
.../org/apache/connect/mongo/MongoFactoryTest.java | 173 +++++++++++++++++++++
.../connect/mongo/MongoSourceConnectorTest.java | 19 +--
.../apache/connect/mongo/MongoSourceTaskTest.java | 143 +++++++++++++++++
.../java/org/apache/connect/mongo/MongoTest.java | 18 +--
.../apache/connect/mongo/ReplicaContextTest.java | 3 +-
.../org/apache/connect/mongo/ReplicaSetTest.java | 59 +++++++
.../org/apache/connect/mongo/ReplicaSetsTest.java | 19 +--
25 files changed, 758 insertions(+), 232 deletions(-)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..fb2c6c6
--- /dev/null
+++ b/README.md
@@ -0,0 +1,12 @@
+# RocketMQ-connect-mongo
+
+this is source connector moudle for mongo,you can run this by running rocketmq connecotr api,
+
+some junit rely on mongo database you can start with a docker container
+
+`docker run -p27027:27017 --name mongo-test -d mongo:4.0.10 --replSet "repl1"`
+
+and then init a mongo replicaSet
+
+`docker exec -it mongo-test mongo ` and `rs.initiate()` and then you can run all junit test
+
diff --git a/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java b/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
index b79b2e9..3df9dc4 100644
--- a/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
+++ b/src/main/java/org/apache/connect/mongo/SourceTaskConfig.java
@@ -1,12 +1,11 @@
package org.apache.connect.mongo;
import io.openmessaging.KeyValue;
-import org.bson.BsonTimestamp;
-
import java.lang.reflect.Method;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
+import org.bson.BsonTimestamp;
public class SourceTaskConfig {
@@ -18,15 +17,50 @@ public class SourceTaskConfig {
private String positionTimeStamp;
private String positionInc;
private String dataSync;
+ private String serverSelectionTimeoutMS;
+ private String connectTimeoutMS;
+ private String socketTimeoutMS;
+ private String ssl;
+ private String tsl;
+ private String tlsInsecure;
+ private String sslInvalidHostNameAllowed;
+ private String tlsAllowInvalidHostnames;
+ private String compressors;
+ private String zlibCompressionLevel;
+ private String trustStore;
+ private String trustStorePassword;
private int copyThread = Runtime.getRuntime().availableProcessors();
-
public static final Set<String> REQUEST_CONFIG = Collections.unmodifiableSet(new HashSet<String>() {
{
add("mongoAddr");
}
});
+ public String getTrustStore() {
+ return trustStore;
+ }
+
+ public void setTrustStore(String trustStore) {
+ this.trustStore = trustStore;
+ }
+
+ public String getTrustStorePassword() {
+ return trustStorePassword;
+ }
+
+ public void setTrustStorePassword(String trustStorePassword) {
+ this.trustStorePassword = trustStorePassword;
+ }
+
+ public String getZlibCompressionLevel() {
+ return zlibCompressionLevel;
+ }
+
+ public void setZlibCompressionLevel(String zlibCompressionLevel) {
+ this.zlibCompressionLevel = zlibCompressionLevel;
+ }
+
public String getPositionInc() {
return positionInc;
}
@@ -67,7 +101,6 @@ public class SourceTaskConfig {
this.mongoAddr = mongoAddr;
}
-
public String getMongoUserName() {
return mongoUserName;
}
@@ -84,7 +117,6 @@ public class SourceTaskConfig {
this.mongoPassWord = mongoPassWord;
}
-
public String getDataSync() {
return dataSync;
}
@@ -93,15 +125,86 @@ public class SourceTaskConfig {
this.dataSync = dataSync;
}
-
public String getReplicaSet() {
return replicaSet;
}
+ public String getServerSelectionTimeoutMS() {
+ return serverSelectionTimeoutMS;
+ }
+
+ public void setServerSelectionTimeoutMS(String serverSelectionTimeoutMS) {
+ this.serverSelectionTimeoutMS = serverSelectionTimeoutMS;
+ }
+
public void setReplicaSet(String replicaSet) {
this.replicaSet = replicaSet;
}
+ public String getConnectTimeoutMS() {
+ return connectTimeoutMS;
+ }
+
+ public void setConnectTimeoutMS(String connectTimeoutMS) {
+ this.connectTimeoutMS = connectTimeoutMS;
+ }
+
+ public String getSocketTimeoutMS() {
+ return socketTimeoutMS;
+ }
+
+ public void setSocketTimeoutMS(String socketTimeoutMS) {
+ this.socketTimeoutMS = socketTimeoutMS;
+ }
+
+ public String getSsl() {
+ return ssl;
+ }
+
+ public void setSsl(String ssl) {
+ this.ssl = ssl;
+ }
+
+ public String getTsl() {
+ return tsl;
+ }
+
+ public void setTsl(String tsl) {
+ this.tsl = tsl;
+ }
+
+ public String getTlsInsecure() {
+ return tlsInsecure;
+ }
+
+ public void setTlsInsecure(String tlsInsecure) {
+ this.tlsInsecure = tlsInsecure;
+ }
+
+ public String getSslInvalidHostNameAllowed() {
+ return sslInvalidHostNameAllowed;
+ }
+
+ public void setSslInvalidHostNameAllowed(String sslInvalidHostNameAllowed) {
+ this.sslInvalidHostNameAllowed = sslInvalidHostNameAllowed;
+ }
+
+ public String getTlsAllowInvalidHostnames() {
+ return tlsAllowInvalidHostnames;
+ }
+
+ public void setTlsAllowInvalidHostnames(String tlsAllowInvalidHostnames) {
+ this.tlsAllowInvalidHostnames = tlsAllowInvalidHostnames;
+ }
+
+ public String getCompressors() {
+ return compressors;
+ }
+
+ public void setCompressors(String compressors) {
+ this.compressors = compressors;
+ }
+
public void load(KeyValue props) {
properties2Object(props, this);
@@ -148,7 +251,6 @@ public class SourceTaskConfig {
}
}
-
public BsonTimestamp getPosition() {
return new BsonTimestamp(Integer.valueOf(positionTimeStamp), Integer.valueOf(positionInc));
}
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
index 2b28ea2..e3dfb6f 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceConnector.java
@@ -3,19 +3,17 @@ package org.apache.connect.mongo.connector;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.source.SourceConnector;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.connect.mongo.SourceTaskConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-
public class MongoSourceConnector extends SourceConnector {
private Logger logger = LoggerFactory.getLogger(this.getClass());
private KeyValue keyValueConfig;
-
@Override
public String verifyAndSetConfig(KeyValue config) {
for (String requestKey : SourceTaskConfig.REQUEST_CONFIG) {
diff --git a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
index 407608a..da244cd 100644
--- a/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
+++ b/src/main/java/org/apache/connect/mongo/connector/MongoSourceTask.java
@@ -4,16 +4,19 @@ import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.data.SourceDataEntry;
import io.openmessaging.connector.api.source.SourceTask;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.connect.mongo.SourceTaskConfig;
-import org.apache.connect.mongo.replicator.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.SourceTaskConfig;
+import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.replicator.ReplicaSet;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.ReplicaSets;
+import org.apache.connect.mongo.replicator.ReplicaSetsContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class MongoSourceTask extends SourceTask {
@@ -42,7 +45,7 @@ public class MongoSourceTask extends SourceTask {
replicaSets = ReplicaSets.create(sourceTaskConfig.getMongoAddr());
replicaSets.getReplicaConfigByName().forEach((replicaSetName, replicaSetConfig) -> {
ByteBuffer byteBuffer = this.context.positionStorageReader().getPosition(ByteBuffer.wrap(
- replicaSetName.getBytes()));
+ replicaSetName.getBytes()));
if (byteBuffer != null && byteBuffer.array().length > 0) {
String positionJson = new String(byteBuffer.array(), StandardCharsets.UTF_8);
ReplicaSetConfig.Position position = JSONObject.parseObject(positionJson, ReplicaSetConfig.Position.class);
@@ -50,11 +53,11 @@ public class MongoSourceTask extends SourceTask {
} else {
ReplicaSetConfig.Position position = replicaSetConfig.emptyPosition();
position.setTimeStamp(sourceTaskConfig.getPositionTimeStamp() != null
- && pattern.matcher(sourceTaskConfig.getPositionTimeStamp()).matches()
- ? Integer.valueOf(sourceTaskConfig.getPositionTimeStamp()) : 0);
+ && pattern.matcher(sourceTaskConfig.getPositionTimeStamp()).matches()
+ ? Integer.valueOf(sourceTaskConfig.getPositionTimeStamp()) : 0);
position.setInc(sourceTaskConfig.getPositionInc() != null
- && pattern.matcher(sourceTaskConfig.getPositionInc()).matches()
- ? Integer.valueOf(sourceTaskConfig.getPositionInc()) : 0);
+ && pattern.matcher(sourceTaskConfig.getPositionInc()).matches()
+ ? Integer.valueOf(sourceTaskConfig.getPositionInc()) : 0);
position.setInitSync(StringUtils.equals(sourceTaskConfig.getDataSync(), Constants.INITSYNC) ? true : false);
replicaSetConfig.setPosition(position);
}
@@ -64,7 +67,6 @@ public class MongoSourceTask extends SourceTask {
replicaSet.start();
});
-
} catch (Throwable throwable) {
logger.error("task start error", throwable);
stop();
diff --git a/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java b/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
index 7c0db7d..87d92ea 100644
--- a/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
+++ b/src/main/java/org/apache/connect/mongo/connector/builder/MongoDataEntry.java
@@ -1,19 +1,27 @@
package org.apache.connect.mongo.connector.builder;
-
import com.alibaba.fastjson.JSONObject;
-import io.openmessaging.connector.api.data.*;
+import io.openmessaging.connector.api.data.DataEntryBuilder;
+import io.openmessaging.connector.api.data.Field;
+import io.openmessaging.connector.api.data.FieldType;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.SourceDataEntry;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import org.apache.connect.mongo.replicator.Constants;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
import org.bson.BsonTimestamp;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-
-import static org.apache.connect.mongo.replicator.Constants.*;
+import static org.apache.connect.mongo.replicator.Constants.CREATED;
+import static org.apache.connect.mongo.replicator.Constants.NAMESPACE;
+import static org.apache.connect.mongo.replicator.Constants.OBJECTID;
+import static org.apache.connect.mongo.replicator.Constants.OPERATIONTYPE;
+import static org.apache.connect.mongo.replicator.Constants.PATCH;
+import static org.apache.connect.mongo.replicator.Constants.TIMESTAMP;
+import static org.apache.connect.mongo.replicator.Constants.VERSION;
public class MongoDataEntry {
@@ -28,8 +36,8 @@ public class MongoDataEntry {
Schema schema = createdSchema(replicaSetConfig.getReplicaSetName());
dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(event.getNamespace().replace(".", "-").replace("$", "-"))
- .entryType(event.getEntryType());
+ .queue(event.getNamespace().replace(".", "-").replace("$", "-"))
+ .entryType(event.getEntryType());
dataEntryBuilder.putFiled(CREATED, event.getDocument().toJson());
dataEntryBuilder.putFiled(NAMESPACE, event.getNamespace());
@@ -38,8 +46,8 @@ public class MongoDataEntry {
Schema schema = oplogSchema(replicaSetConfig.getReplicaSetName());
dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(event.getNamespace().replace(".", "-").replace("$", "-"))
- .entryType(event.getEntryType());
+ .queue(event.getNamespace().replace(".", "-").replace("$", "-"))
+ .entryType(event.getEntryType());
dataEntryBuilder.putFiled(OPERATIONTYPE, event.getOperationType().name());
dataEntryBuilder.putFiled(TIMESTAMP, event.getTimestamp().getValue());
dataEntryBuilder.putFiled(VERSION, event.getV());
@@ -48,15 +56,13 @@ public class MongoDataEntry {
dataEntryBuilder.putFiled(OBJECTID, event.getObjectId().isPresent() ? JSONObject.toJSONString(event.getObjectId().get()) : "");
}
-
String position = createPosition(event, replicaSetConfig);
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap(replicaSetConfig.getReplicaSetName().getBytes(StandardCharsets.UTF_8)),
- ByteBuffer.wrap(position.getBytes(StandardCharsets.UTF_8)));
+ ByteBuffer.wrap(replicaSetConfig.getReplicaSetName().getBytes(StandardCharsets.UTF_8)),
+ ByteBuffer.wrap(position.getBytes(StandardCharsets.UTF_8)));
return sourceDataEntry;
}
-
private static String createPosition(ReplicationEvent event, ReplicaSetConfig replicaSetConfig) {
ReplicaSetConfig.Position position = replicaSetConfig.emptyPosition();
BsonTimestamp timestamp = event.getTimestamp();
@@ -76,7 +82,6 @@ public class MongoDataEntry {
return schema;
}
-
private static Schema oplogSchema(String dataSourceName) {
Schema schema = new Schema();
schema.setDataSource(dataSourceName);
@@ -85,7 +90,6 @@ public class MongoDataEntry {
return schema;
}
-
private static void createdField(Schema schema) {
Field namespace = new Field(0, NAMESPACE, FieldType.STRING);
schema.getFields().add(namespace);
@@ -109,5 +113,4 @@ public class MongoDataEntry {
schema.getFields().add(objectId);
}
-
}
diff --git a/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java b/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
index 018418c..9c73eac 100644
--- a/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
+++ b/src/main/java/org/apache/connect/mongo/initsync/CollectionMeta.java
@@ -34,8 +34,8 @@ public class CollectionMeta {
@Override
public String toString() {
return "CollectionMeta{" +
- "databaseName='" + databaseName + '\'' +
- ", collectionName='" + collectionName + '\'' +
- '}';
+ "databaseName='" + databaseName + '\'' +
+ ", collectionName='" + collectionName + '\'' +
+ '}';
}
}
diff --git a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
index d92e968..6cdbe06 100644
--- a/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
+++ b/src/main/java/org/apache/connect/mongo/initsync/InitSync.java
@@ -3,6 +3,13 @@ package org.apache.connect.mongo.initsync;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoIterable;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.connect.mongo.replicator.ReplicaSet;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
import org.apache.connect.mongo.replicator.ReplicaSetsContext;
@@ -13,14 +20,6 @@ import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicInteger;
-
public class InitSync {
private Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -34,7 +33,8 @@ public class InitSync {
private CountDownLatch countDownLatch;
private ReplicaSet replicaSet;
- public InitSync(ReplicaSetConfig replicaSetConfig, MongoClient mongoClient, ReplicaSetsContext context, ReplicaSet replicaSet) {
+ public InitSync(ReplicaSetConfig replicaSetConfig, MongoClient mongoClient, ReplicaSetsContext context,
+ ReplicaSet replicaSet) {
this.replicaSetConfig = replicaSetConfig;
this.mongoClient = mongoClient;
this.context = context;
@@ -89,7 +89,6 @@ public class InitSync {
}
-
class CopyRunner implements Runnable {
private MongoClient mongoClient;
@@ -97,7 +96,8 @@ public class InitSync {
private CollectionMeta collectionMeta;
private ReplicaSet replicaSet;
- public CopyRunner(MongoClient mongoClient, CountDownLatch countDownLatch, CollectionMeta collectionMeta, ReplicaSet replicaSet) {
+ public CopyRunner(MongoClient mongoClient, CountDownLatch countDownLatch, CollectionMeta collectionMeta,
+ ReplicaSet replicaSet) {
this.mongoClient = mongoClient;
this.countDownLatch = countDownLatch;
this.collectionMeta = collectionMeta;
@@ -110,10 +110,10 @@ public class InitSync {
int count = 0;
try {
MongoCursor<Document> mongoCursor = mongoClient.getDatabase(collectionMeta.getDatabaseName())
- .getCollection(collectionMeta.getCollectionName())
- .find()
- .batchSize(200)
- .iterator();
+ .getCollection(collectionMeta.getCollectionName())
+ .find()
+ .batchSize(200)
+ .iterator();
while (replicaSet.isRuning() && mongoCursor.hasNext()) {
if (context.initSyncAbort()) {
logger.info("init sync database:{}, collection:{} abort, has copy:{} document", collectionMeta.getDatabaseName(), collectionMeta.getCollectionName(), count);
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Constants.java b/src/main/java/org/apache/connect/mongo/replicator/Constants.java
index 1a91a57..c895bd6 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/Constants.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/Constants.java
@@ -2,13 +2,9 @@ package org.apache.connect.mongo.replicator;
public class Constants {
-
-
public static final String MONGO_LOCAL_DATABASE = "local";
public static final String MONGO_OPLOG_RS = "oplog.rs";
-
-
public static final String OPERATIONTYPE = "op";
public static final String TIMESTAMP = "ts";
public static final String VERSION = "v";
@@ -17,11 +13,9 @@ public class Constants {
public static final String OPERATION = "o";
public static final String OBJECTID = "o2";
-
public static final String CREATED = "created";
public static final String PATCH = "patch";
-
public static final String INITSYNC = "initSync";
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/Filter.java b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
index 3e431a3..fd26163 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/Filter.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/Filter.java
@@ -1,26 +1,23 @@
package org.apache.connect.mongo.replicator;
-
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.TypeReference;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
import org.apache.commons.lang3.StringUtils;
import org.apache.connect.mongo.SourceTaskConfig;
import org.apache.connect.mongo.initsync.CollectionMeta;
import org.apache.connect.mongo.replicator.event.OperationType;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-
public class Filter {
private Function<CollectionMeta, Boolean> dbAndCollectionFilter;
private Map<String, List<String>> interestMap = new HashMap<>();
private Function<OperationType, Boolean> notNoopFilter;
-
public Filter(SourceTaskConfig sourceTaskConfig) {
String interestDbAndCollection = sourceTaskConfig.getInterestDbAndCollection();
@@ -33,7 +30,6 @@ public class Filter {
interestMap.put(db, collections);
}
-
}
dbAndCollectionFilter = (collectionMeta) -> {
@@ -56,13 +52,12 @@ public class Filter {
notNoopFilter = (opeartionType) -> opeartionType.ordinal() != OperationType.NOOP.ordinal();
}
-
public boolean filterMeta(CollectionMeta collectionMeta) {
return dbAndCollectionFilter.apply(collectionMeta);
}
public boolean filterEvent(ReplicationEvent event) {
return dbAndCollectionFilter.apply(new CollectionMeta(event.getDatabaseName(), event.getCollectionName()))
- && notNoopFilter.apply(event.getOperationType());
+ && notNoopFilter.apply(event.getOperationType());
}
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java b/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
new file mode 100644
index 0000000..f5e01a3
--- /dev/null
+++ b/src/main/java/org/apache/connect/mongo/replicator/MongoClientFactory.java
@@ -0,0 +1,113 @@
+package org.apache.connect.mongo.replicator;
+
+import com.mongodb.ConnectionString;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import java.util.Properties;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.SourceTaskConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MongoClientFactory {
+
+ private Logger logger = LoggerFactory.getLogger(this.getClass());
+
+ private SourceTaskConfig taskConfig;
+
+ public MongoClientFactory(SourceTaskConfig sourceTaskConfig) {
+ this.taskConfig = sourceTaskConfig;
+ }
+
+ public MongoClient createMongoClient(ReplicaSetConfig replicaSetConfig) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("mongodb://");
+ if (StringUtils.isNotBlank(taskConfig.getMongoUserName())
+ && StringUtils.isNotBlank(taskConfig.getMongoPassWord())) {
+ sb.append(taskConfig.getMongoUserName());
+ sb.append(":");
+ sb.append(taskConfig.getMongoPassWord());
+ sb.append("@");
+
+ }
+ sb.append(replicaSetConfig.getHost());
+ sb.append("/");
+ if (StringUtils.isNotBlank(replicaSetConfig.getReplicaSetName())) {
+ sb.append("?");
+ sb.append("replicaSet=");
+ sb.append(replicaSetConfig.getReplicaSetName());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getServerSelectionTimeoutMS())) {
+ sb.append("&");
+ sb.append("serverSelectionTimeoutMS=");
+ sb.append(taskConfig.getServerSelectionTimeoutMS());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getConnectTimeoutMS())) {
+ sb.append("&");
+ sb.append("connectTimeoutMS=");
+ sb.append(taskConfig.getConnectTimeoutMS());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getSocketTimeoutMS())) {
+ sb.append("&");
+ sb.append("socketTimeoutMS=");
+ sb.append(taskConfig.getSocketTimeoutMS());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getSsl()) || StringUtils.isNotBlank(taskConfig.getTsl())) {
+ sb.append("&");
+ sb.append("ssl=");
+ sb.append(true);
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getTlsInsecure())) {
+ sb.append("&");
+ sb.append("tlsInsecure=");
+ sb.append(taskConfig.getTlsInsecure());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getTlsAllowInvalidHostnames())) {
+ sb.append("&");
+ sb.append("tlsAllowInvalidHostnames=");
+ sb.append(taskConfig.getTlsAllowInvalidHostnames());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getSslInvalidHostNameAllowed())) {
+ sb.append("&");
+ sb.append("sslInvalidHostNameAllowed=");
+ sb.append(taskConfig.getSslInvalidHostNameAllowed());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getCompressors())) {
+ sb.append("&");
+ sb.append("compressors=");
+ sb.append(taskConfig.getCompressors());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getZlibCompressionLevel())) {
+ sb.append("&");
+ sb.append("zlibcompressionlevel=");
+ sb.append(taskConfig.getZlibCompressionLevel());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getTrustStore())) {
+ Properties properties = System.getProperties();
+ properties.put("javax.net.ssl.trustStore", taskConfig.getTrustStore());
+ logger.info("javax.net.ssl.trustStore: {}", taskConfig.getTrustStore());
+ }
+
+ if (StringUtils.isNotBlank(taskConfig.getTrustStorePassword())) {
+ Properties properties = System.getProperties();
+ properties.put("javax.net.ssl.trustStorePassword", taskConfig.getTrustStorePassword());
+ logger.info("javax.net.ssl.trustStorePassword: {}", taskConfig.getTrustStorePassword());
+ }
+
+ logger.info("connection string :{}", sb.toString());
+ System.out.println(sb.toString());
+ ConnectionString connectionString = new ConnectionString(sb.toString());
+ return MongoClients.create(connectionString);
+ }
+
+}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
index b141c2b..8f4d0d8 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSet.java
@@ -4,18 +4,16 @@ import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.MongoIterable;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.connect.mongo.replicator.Constants.MONGO_LOCAL_DATABASE;
import static org.apache.connect.mongo.replicator.Constants.MONGO_OPLOG_RS;
-
public class ReplicaSet {
private Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -24,7 +22,6 @@ public class ReplicaSet {
private ReplicaSetConfig replicaSetConfig;
-
private ReplicaSetsContext replicaSetsContext;
private MongoClient mongoClient;
@@ -56,7 +53,6 @@ public class ReplicaSet {
}
}
-
public boolean isReplicaMongo() {
MongoDatabase local = mongoClient.getDatabase(MONGO_LOCAL_DATABASE);
MongoIterable<String> collectionNames = local.listCollectionNames();
@@ -82,7 +78,6 @@ public class ReplicaSet {
}
-
public void pause() {
pause = true;
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
index 4b8d148..1b54b17 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetConfig.java
@@ -1,18 +1,15 @@
package org.apache.connect.mongo.replicator;
-import org.bson.BsonTimestamp;
-
import java.util.Objects;
+import org.bson.BsonTimestamp;
public class ReplicaSetConfig {
-
private String shardName;
private String replicaSetName;
private String host;
private Position position;
-
public Position getPosition() {
return position;
}
@@ -55,13 +52,21 @@ public class ReplicaSetConfig {
return new Position(0, 0, true);
}
+ @Override
+ public String toString() {
+ return "ReplicaSetConfig{" +
+ "shardName='" + shardName + '\'' +
+ ", replicaSetName='" + replicaSetName + '\'' +
+ ", host='" + host + '\'' +
+ ", position=" + position +
+ '}';
+ }
public class Position {
private int timeStamp;
private int inc;
private boolean initSync;
-
public int getTimeStamp() {
return timeStamp;
}
@@ -86,7 +91,6 @@ public class ReplicaSetConfig {
this.initSync = initSync;
}
-
public Position(int timeStamp, int inc, boolean initSync) {
this.timeStamp = timeStamp;
this.inc = inc;
@@ -104,20 +108,22 @@ public class ReplicaSetConfig {
@Override
public String toString() {
return "Position{" +
- "timeStamp=" + timeStamp +
- ", inc=" + inc +
- ", initSync=" + initSync +
- '}';
+ "timeStamp=" + timeStamp +
+ ", inc=" + inc +
+ ", initSync=" + initSync +
+ '}';
}
@Override
public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
Position position = (Position) o;
return timeStamp == position.timeStamp &&
- inc == position.inc &&
- initSync == position.initSync;
+ inc == position.inc &&
+ initSync == position.initSync;
}
@Override
@@ -125,15 +131,4 @@ public class ReplicaSetConfig {
return Objects.hash(timeStamp, inc, initSync);
}
}
-
-
- @Override
- public String toString() {
- return "ReplicaSetConfig{" +
- "shardName='" + shardName + '\'' +
- ", replicaSetName='" + replicaSetName + '\'' +
- ", host='" + host + '\'' +
- ", position=" + position +
- '}';
- }
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java
index 9184b90..af1ebeb 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSets.java
@@ -1,23 +1,20 @@
package org.apache.connect.mongo.replicator;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.Validate;
-
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.Validate;
public class ReplicaSets {
private static final Pattern HOST_PATTERN = Pattern.compile("((([^=]+)[=])?(([^/]+)\\/))?(.+)");
-
private final Map<String, ReplicaSetConfig> replicaConfigByName = new HashMap<>();
-
public ReplicaSets(Set<ReplicaSetConfig> replicaSetConfigs) {
replicaSetConfigs.forEach(replicaSetConfig -> {
if (StringUtils.isNotBlank(replicaSetConfig.getReplicaSetName())) {
@@ -43,7 +40,6 @@ public class ReplicaSets {
return new ReplicaSets(replicaSetConfigs);
}
-
private static ReplicaSetConfig parseReplicaSetStr(String hosts) {
if (hosts != null) {
Matcher matcher = HOST_PATTERN.matcher(hosts);
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
index f66ca14..e599f5b 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicaSetsContext.java
@@ -1,15 +1,7 @@
package org.apache.connect.mongo.replicator;
-import com.mongodb.ConnectionString;
import com.mongodb.client.MongoClient;
-import com.mongodb.client.MongoClients;
import io.openmessaging.connector.api.data.SourceDataEntry;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.connect.mongo.SourceTaskConfig;
-import org.apache.connect.mongo.connector.builder.MongoDataEntry;
-import org.apache.connect.mongo.initsync.CollectionMeta;
-import org.apache.connect.mongo.replicator.event.ReplicationEvent;
-
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -17,6 +9,10 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.connect.mongo.SourceTaskConfig;
+import org.apache.connect.mongo.connector.builder.MongoDataEntry;
+import org.apache.connect.mongo.initsync.CollectionMeta;
+import org.apache.connect.mongo.replicator.event.ReplicationEvent;
public class ReplicaSetsContext {
@@ -30,57 +26,36 @@ public class ReplicaSetsContext {
private Filter filter;
+ private MongoClientFactory mongoClientFactory;
+
public ReplicaSetsContext(SourceTaskConfig taskConfig) {
this.taskConfig = taskConfig;
this.replicaSets = new CopyOnWriteArrayList<>();
this.dataEntryQueue = new LinkedBlockingDeque<>();
this.filter = new Filter(taskConfig);
+ this.mongoClientFactory = new MongoClientFactory(taskConfig);
}
-
public MongoClient createMongoClient(ReplicaSetConfig replicaSetConfig) {
- StringBuilder sb = new StringBuilder();
- sb.append("mongodb://");
- if (StringUtils.isNotBlank(taskConfig.getMongoUserName())
- && StringUtils.isNotBlank(taskConfig.getMongoPassWord())) {
- sb.append(taskConfig.getMongoUserName());
- sb.append(":");
- sb.append(taskConfig.getMongoPassWord());
- sb.append("@");
-
- }
- sb.append(replicaSetConfig.getHost());
- sb.append("/");
- if (StringUtils.isNotBlank(replicaSetConfig.getReplicaSetName())) {
- sb.append("?");
- sb.append("replicaSet=");
- sb.append(replicaSetConfig.getReplicaSetName());
- }
- ConnectionString connectionString = new ConnectionString(sb.toString());
- return MongoClients.create(connectionString);
+ return mongoClientFactory.createMongoClient(replicaSetConfig);
}
-
public boolean filterEvent(ReplicationEvent event) {
return filter.filterEvent(event);
}
-
public boolean filterMeta(CollectionMeta collectionMeta) {
return filter.filterMeta(collectionMeta);
}
-
public int getCopyThread() {
return taskConfig.getCopyThread() > 0 ? taskConfig.getCopyThread() : Runtime.getRuntime().availableProcessors();
}
-
public void addReplicaSet(ReplicaSet replicaSet) {
this.replicaSets.add(replicaSet);
}
-
public void shutdown() {
replicaSets.forEach(ReplicaSet::shutdown);
}
@@ -89,12 +64,10 @@ public class ReplicaSetsContext {
replicaSets.forEach(ReplicaSet::pause);
}
-
public void resume() {
replicaSets.forEach(ReplicaSet::resume);
}
-
public void publishEvent(ReplicationEvent event, ReplicaSetConfig replicaSetConfig) {
SourceDataEntry sourceDataEntry = MongoDataEntry.createSouceDataEntry(event, replicaSetConfig);
while (true) {
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
index bf4ebac..6cb46d1 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
@@ -13,7 +13,6 @@ import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
public class ReplicatorTask implements Runnable {
private Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -26,7 +25,8 @@ public class ReplicatorTask implements Runnable {
private ReplicaSetsContext replicaSetsContext;
- public ReplicatorTask(ReplicaSet replicaSet, MongoClient mongoClient, ReplicaSetConfig replicaSetConfig, ReplicaSetsContext replicaSetsContext) {
+ public ReplicatorTask(ReplicaSet replicaSet, MongoClient mongoClient, ReplicaSetConfig replicaSetConfig,
+ ReplicaSetsContext replicaSetsContext) {
this.replicaSet = replicaSet;
this.replicaSetConfig = replicaSetConfig;
this.mongoClient = mongoClient;
@@ -45,15 +45,15 @@ public class ReplicatorTask implements Runnable {
FindIterable<Document> iterable;
if (replicaSetConfig.getPosition().isValid()) {
iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find(
- Filters.gt("ts", replicaSetConfig.getPosition().converBsonTimeStamp()));
+ Filters.gt("ts", replicaSetConfig.getPosition().converBsonTimeStamp()));
} else {
iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
}
MongoCursor<Document> cursor = iterable.sort(new Document("$natural", 1))
- .noCursorTimeout(true)
- .cursorType(CursorType.TailableAwait)
- .batchSize(200)
- .iterator();
+ .noCursorTimeout(true)
+ .cursorType(CursorType.TailableAwait)
+ .batchSize(200)
+ .iterator();
while (replicaSet.isRuning()) {
try {
@@ -70,7 +70,6 @@ public class ReplicatorTask implements Runnable {
logger.info("replicaSet:{}, already shutdown, replicaTask end of life cycle", replicaSetConfig);
}
-
private void executorCursor(MongoCursor<Document> cursor) {
while (cursor.hasNext() && !replicaSet.isPause()) {
Document document = cursor.next();
@@ -81,5 +80,4 @@ public class ReplicatorTask implements Runnable {
}
}
-
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java b/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java
index 57b4ac8..1b48990 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/EventConverter.java
@@ -1,16 +1,19 @@
package org.apache.connect.mongo.replicator.event;
+import java.util.Optional;
import org.bson.BsonTimestamp;
import org.bson.Document;
-import java.util.Optional;
-
-import static org.apache.connect.mongo.replicator.Constants.*;
-
+import static org.apache.connect.mongo.replicator.Constants.HASH;
+import static org.apache.connect.mongo.replicator.Constants.NAMESPACE;
+import static org.apache.connect.mongo.replicator.Constants.OBJECTID;
+import static org.apache.connect.mongo.replicator.Constants.OPERATION;
+import static org.apache.connect.mongo.replicator.Constants.OPERATIONTYPE;
+import static org.apache.connect.mongo.replicator.Constants.TIMESTAMP;
+import static org.apache.connect.mongo.replicator.Constants.VERSION;
public class EventConverter {
-
public static ReplicationEvent convert(Document document, String replicaSetName) {
ReplicationEvent event = new ReplicationEvent();
diff --git a/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java b/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
index 283e9d6..6407781 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/event/ReplicationEvent.java
@@ -1,12 +1,11 @@
package org.apache.connect.mongo.replicator.event;
import io.openmessaging.connector.api.data.EntryType;
+import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.bson.BsonTimestamp;
import org.bson.Document;
-import java.util.Optional;
-
public class ReplicationEvent {
private Document document;
@@ -21,13 +20,12 @@ public class ReplicationEvent {
private Optional<Document> objectId;
private String replicaSetName;
-
public ReplicationEvent() {
}
-
- public ReplicationEvent(OperationType operationType, BsonTimestamp timestamp, Integer v, Long h, String namespace, Optional<Document> eventData, Optional<Document> objectId, Document document) {
+ public ReplicationEvent(OperationType operationType, BsonTimestamp timestamp, Integer v, Long h, String namespace,
+ Optional<Document> eventData, Optional<Document> objectId, Document document) {
this.operationType = operationType;
this.v = v;
this.h = h;
@@ -41,7 +39,6 @@ public class ReplicationEvent {
this.document = document;
}
-
public OperationType getOperationType() {
return operationType;
}
@@ -91,12 +88,10 @@ public class ReplicationEvent {
}
}
-
public void setOperationType(OperationType operationType) {
this.operationType = operationType;
}
-
public Document getDocument() {
return document;
}
@@ -137,7 +132,6 @@ public class ReplicationEvent {
this.objectId = objectId;
}
-
public void setReplicaSetName(String replicaSetName) {
this.replicaSetName = replicaSetName;
}
@@ -149,17 +143,17 @@ public class ReplicationEvent {
@Override
public String toString() {
return "ReplicationEvent{" +
- "document=" + document +
- ", operationType=" + operationType +
- ", v=" + v +
- ", h=" + h +
- ", timestamp=" + timestamp +
- ", databaseName='" + databaseName + '\'' +
- ", collectionName='" + collectionName + '\'' +
- ", namespace='" + namespace + '\'' +
- ", eventData=" + eventData +
- ", objectId=" + objectId +
- ", replicaSetName='" + replicaSetName + '\'' +
- '}';
+ "document=" + document +
+ ", operationType=" + operationType +
+ ", v=" + v +
+ ", h=" + h +
+ ", timestamp=" + timestamp +
+ ", databaseName='" + databaseName + '\'' +
+ ", collectionName='" + collectionName + '\'' +
+ ", namespace='" + namespace + '\'' +
+ ", eventData=" + eventData +
+ ", objectId=" + objectId +
+ ", replicaSetName='" + replicaSetName + '\'' +
+ '}';
}
}
diff --git a/src/test/java/org/apache/connect/mongo/FilterTest.java b/src/test/java/org/apache/connect/mongo/FilterTest.java
index 60e2514..a804b8b 100644
--- a/src/test/java/org/apache/connect/mongo/FilterTest.java
+++ b/src/test/java/org/apache/connect/mongo/FilterTest.java
@@ -1,6 +1,10 @@
package org.apache.connect.mongo;
import com.alibaba.fastjson.JSONObject;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.connect.mongo.initsync.CollectionMeta;
import org.apache.connect.mongo.replicator.Filter;
import org.apache.connect.mongo.replicator.event.OperationType;
@@ -9,14 +13,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
public class FilterTest {
-
private SourceTaskConfig sourceTaskConfig;
private Map<String, List<String>> insterest;
@@ -37,7 +35,6 @@ public class FilterTest {
Assert.assertFalse(filter.filterMeta(new CollectionMeta("test", "person01")));
}
-
@Test
public void testBlankDb() {
Filter filter = new Filter(sourceTaskConfig);
@@ -45,7 +42,6 @@ public class FilterTest {
Assert.assertTrue(filter.filterMeta(new CollectionMeta("test1", "test01")));
}
-
@Test
public void testAsterisk() {
List<String> collections = new ArrayList<>();
@@ -57,8 +53,6 @@ public class FilterTest {
Assert.assertTrue(filter.filterMeta(new CollectionMeta("test", "tests032")));
}
-
-
@Test
public void testFilterEvent() {
Filter filter = new Filter(sourceTaskConfig);
@@ -69,5 +63,4 @@ public class FilterTest {
Assert.assertTrue(filter.filterEvent(replicationEvent));
}
-
}
diff --git a/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java b/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
new file mode 100644
index 0000000..93adeeb
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/MongoFactoryTest.java
@@ -0,0 +1,173 @@
+package org.apache.connect.mongo;
+
+import com.mongodb.MongoClientSettings;
+import com.mongodb.MongoTimeoutException;
+import com.mongodb.client.internal.MongoClientImpl;
+import java.lang.reflect.Field;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.replicator.MongoClientFactory;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MongoFactoryTest {
+
+ private ReplicaSetConfig replicaSetConfig;
+
+ private SourceTaskConfig sourceTaskConfig;
+
+ private MongoClientFactory mongoClientFactory;
+
+ private MongoClientImpl client;
+
+ @Before
+ public void before() {
+ this.replicaSetConfig = new ReplicaSetConfig("shardName1", "rep1", "127.0.0.1:27027");
+ this.sourceTaskConfig = new SourceTaskConfig();
+ this.mongoClientFactory = new MongoClientFactory(sourceTaskConfig);
+ }
+
+ @After
+ public void after() {
+ client.close();
+ }
+
+ @Test
+ public void testCreateMongoClientWithSSL() {
+ sourceTaskConfig.setSsl("ssl");
+ MongoClientSettings settings = getSettings();
+ System.out.println(settings.getSslSettings());
+ Assert.assertTrue(settings.getSslSettings().isEnabled());
+ }
+
+ @Test
+ public void testCreateMongoClientWithTSL() {
+ sourceTaskConfig.setTsl("tsl");
+ MongoClientSettings settings = getSettings();
+ System.out.println(settings.getSslSettings());
+ Assert.assertTrue(settings.getSslSettings().isEnabled());
+ }
+
+ @Test
+ public void testCreateMongoClientWithserverSelectionTimeoutMS() {
+ try {
+ replicaSetConfig.setReplicaSetName("testReplicatSet");
+ sourceTaskConfig.setServerSelectionTimeoutMS("150");
+ System.out.println(getSettings().getClusterSettings());
+ Assert.assertTrue(getSettings().getClusterSettings().getServerSelectionTimeout(TimeUnit.MILLISECONDS) == 150);
+ } catch (MongoTimeoutException exception) {
+ Assert.assertTrue(StringUtils.startsWith(exception.getMessage(), "Timed out after 100 ms while waiting for a server that matches"));
+ }
+ }
+
+ @Test
+ public void testCreateMongoClientWithConnectTimeoutMS() {
+ sourceTaskConfig.setConnectTimeoutMS("1200");
+ System.out.println(getSettings().getSocketSettings());
+ Assert.assertTrue(getSettings().getSocketSettings().getConnectTimeout(TimeUnit.MILLISECONDS) == 1200);
+
+ }
+
+ @Test
+ public void testCreateMongoClientWithSocketTimeoutMS() {
+ sourceTaskConfig.setSocketTimeoutMS("1100");
+ System.out.println(getSettings().getSocketSettings());
+ Assert.assertTrue(getSettings().getSocketSettings().getReadTimeout(TimeUnit.MILLISECONDS) == 1100);
+ }
+
+ @Test
+ public void testCreateMongoClientWithInvalidHostNameAllowed() {
+ sourceTaskConfig.setSslInvalidHostNameAllowed("true");
+ System.out.println(getSettings().getSslSettings());
+ Assert.assertTrue(getSettings().getSslSettings().isInvalidHostNameAllowed());
+
+ sourceTaskConfig.setSslInvalidHostNameAllowed("false");
+ System.out.println(getSettings().getSslSettings());
+ Assert.assertFalse(getSettings().getSslSettings().isInvalidHostNameAllowed());
+ }
+
+ @Test
+ public void testCreateMongoClientWithInvalidHostNameAllowedTsl() {
+ sourceTaskConfig.setTlsAllowInvalidHostnames("true");
+ System.out.println(getSettings().getSslSettings());
+ Assert.assertTrue(getSettings().getSslSettings().isInvalidHostNameAllowed());
+
+ sourceTaskConfig.setTlsAllowInvalidHostnames("false");
+ System.out.println(getSettings().getSslSettings());
+ Assert.assertFalse(getSettings().getSslSettings().isInvalidHostNameAllowed());
+ }
+
+ @Test
+ public void testCreateMongoClientWithTlsinsecure() {
+ sourceTaskConfig.setTlsInsecure("true");
+ System.out.println(getSettings().getSslSettings());
+ Assert.assertTrue(getSettings().getSslSettings().isInvalidHostNameAllowed());
+
+ sourceTaskConfig.setTlsInsecure("false");
+ System.out.println(getSettings().getSslSettings());
+ Assert.assertFalse(getSettings().getSslSettings().isInvalidHostNameAllowed());
+ }
+
+ @Test
+ public void testCreateMongoClientWithCompression() {
+ sourceTaskConfig.setCompressors("zlib");
+ System.out.println(getSettings().getCompressorList());
+ Assert.assertTrue(getSettings().getCompressorList().get(0).getName().equals("zlib"));
+ }
+
+ @Test
+ public void testCreateMongoClientWithCompressionLevel() {
+ sourceTaskConfig.setCompressors("zlib");
+ sourceTaskConfig.setZlibCompressionLevel("7");
+ System.out.println(getSettings().getCompressorList());
+ Assert.assertTrue(getSettings().getCompressorList().get(0).getName().equals("zlib"));
+ Assert.assertTrue(getSettings().getCompressorList().get(0).getProperty("level", 0) == 7);
+ }
+
+ @Test
+ public void testCreateMongoClientWithAuth() {
+ sourceTaskConfig.setMongoUserName("test");
+ sourceTaskConfig.setMongoPassWord("123456");
+ System.out.println(getSettings().getCredential());
+ Assert.assertTrue(getSettings().getCredential().getSource().equals("admin"));
+ Assert.assertTrue(getSettings().getCredential().getUserName().equals("test"));
+ Assert.assertTrue(new String(getSettings().getCredential().getPassword()).equals("123456"));
+ }
+
+ private MongoClientSettings getSettings() {
+ try {
+ client = (MongoClientImpl) mongoClientFactory.createMongoClient(replicaSetConfig);
+ Field field = MongoClientImpl.class.getDeclaredField("settings");
+ field.setAccessible(true);
+ return (MongoClientSettings) field.get(client);
+ } catch (Exception e) {
+
+ }
+ return null;
+ }
+
+// @Test
+// public void testSSLTrustStore() {
+// sourceTaskConfig.setMongoUserName("user_test");
+// sourceTaskConfig.setMongoPassWord("pwd_test");
+// sourceTaskConfig.setSsl("ssl");
+// sourceTaskConfig.setSslInvalidHostNameAllowed("true");
+// sourceTaskConfig.setTrustStore("/Users/liping/test.pem");
+// sourceTaskConfig.setTrustStorePassword("test001");
+// sourceTaskConfig.setServerSelectionTimeoutMS("10000");
+// MongoClient client = mongoClientFactory.createMongoClient(replicaSetConfig);
+// MongoCollection<Document> collection = client.getDatabase("test").getCollection("person");
+// Document document = new Document();
+// document.put("name", "liping");
+// collection.insertOne(document);
+// MongoCursor<Document> iterator = collection.find().iterator();
+// while (iterator.hasNext()) {
+// System.out.println(iterator.next());
+// }
+//
+// }
+
+}
diff --git a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
index 1f6227b..f85e4a9 100644
--- a/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoSourceConnectorTest.java
@@ -5,6 +5,11 @@ import io.openmessaging.connector.api.data.EntryType;
import io.openmessaging.connector.api.data.Schema;
import io.openmessaging.connector.api.data.SourceDataEntry;
import io.openmessaging.internal.DefaultKeyValue;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.LinkedBlockingQueue;
import org.apache.connect.mongo.connector.MongoSourceConnector;
import org.apache.connect.mongo.connector.MongoSourceTask;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
@@ -17,12 +22,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.lang.reflect.Field;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.LinkedBlockingQueue;
-
public class MongoSourceConnectorTest {
private MongoSourceConnector mongoSourceConnector;
@@ -42,14 +41,12 @@ public class MongoSourceConnectorTest {
Assert.assertEquals(mongoSourceConnector.taskClass(), MongoSourceTask.class);
}
-
@Test
public void verifyConfig() {
String s = mongoSourceConnector.verifyAndSetConfig(keyValue);
Assert.assertTrue(s.contains("Request config key:"));
}
-
@Test
public void testPoll() throws Exception {
LinkedBlockingQueue<SourceDataEntry> entries = new LinkedBlockingQueue<>();
@@ -65,25 +62,22 @@ public class MongoSourceConnectorTest {
event.setH(324243242L);
event.setEventData(Optional.ofNullable(new Document("testEventKey", "testEventValue")));
event.setObjectId(Optional.empty());
- context.publishEvent(event, new ReplicaSetConfig("", "testReplicaName", "localhost:27017"));
+ context.publishEvent(event, new ReplicaSetConfig("", "testReplicaName", "localhost:27027"));
List<SourceDataEntry> sourceDataEntries = (List<SourceDataEntry>) context.poll();
Assert.assertTrue(sourceDataEntries.size() == 1);
SourceDataEntry sourceDataEntry = sourceDataEntries.get(0);
Assert.assertEquals("test-person", sourceDataEntry.getQueueName());
-
ByteBuffer sourcePartition = sourceDataEntry.getSourcePartition();
Assert.assertEquals("testReplicaName", new String(sourcePartition.array()));
-
ByteBuffer sourcePosition = sourceDataEntry.getSourcePosition();
ReplicaSetConfig.Position position = JSONObject.parseObject(new String(sourcePosition.array()), ReplicaSetConfig.Position.class);
Assert.assertEquals(position.getTimeStamp(), 1565609506);
Assert.assertEquals(position.getInc(), 1);
Assert.assertEquals(position.isInitSync(), false);
-
EntryType entryType = sourceDataEntry.getEntryType();
Assert.assertEquals(EntryType.CREATE, entryType);
@@ -97,5 +91,4 @@ public class MongoSourceConnectorTest {
}
-
}
diff --git a/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java b/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
new file mode 100644
index 0000000..b696393
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/MongoSourceTaskTest.java
@@ -0,0 +1,143 @@
+package org.apache.connect.mongo;
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.PositionStorageReader;
+import io.openmessaging.connector.api.source.SourceTask;
+import io.openmessaging.connector.api.source.SourceTaskContext;
+import io.openmessaging.internal.DefaultKeyValue;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.connect.mongo.connector.MongoSourceTask;
+import org.apache.connect.mongo.replicator.Constants;
+import org.apache.connect.mongo.replicator.ReplicaSet;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.ReplicaSetsContext;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MongoSourceTaskTest {
+
+ @Test
+ public void testEmptyContextStart() throws NoSuchFieldException, IllegalAccessException {
+ MongoSourceTask mongoSourceTask = new MongoSourceTask();
+ DefaultKeyValue defaultKeyValue = new DefaultKeyValue();
+ defaultKeyValue.put("mongoAddr", "test/127.0.0.1:27027");
+ defaultKeyValue.put("positionTimeStamp", "11111111");
+ defaultKeyValue.put("positionInc", "111");
+ defaultKeyValue.put("serverSelectionTimeoutMS", "10");
+ defaultKeyValue.put("dataSync", Constants.INITSYNC);
+
+ Field context = SourceTask.class.getDeclaredField("context");
+ context.setAccessible(true);
+ context.set(mongoSourceTask, emptyTaskContext());
+ mongoSourceTask.start(defaultKeyValue);
+
+ Field replicaSetsContext = MongoSourceTask.class.getDeclaredField("replicaSetsContext");
+ replicaSetsContext.setAccessible(true);
+ ReplicaSetsContext setsContext = (ReplicaSetsContext) replicaSetsContext.get(mongoSourceTask);
+
+ Field replicaSets = ReplicaSetsContext.class.getDeclaredField("replicaSets");
+ replicaSets.setAccessible(true);
+ List<ReplicaSet> replicaSetList = (List<ReplicaSet>) replicaSets.get(setsContext);
+ Assert.assertTrue(replicaSetList.size() == 1);
+ ReplicaSet replicaSet = replicaSetList.get(0);
+ Field replicaSetConfig = ReplicaSet.class.getDeclaredField("replicaSetConfig");
+ replicaSetConfig.setAccessible(true);
+ ReplicaSetConfig replicaSetConfig1 = (ReplicaSetConfig) replicaSetConfig.get(replicaSet);
+ Assert.assertTrue(StringUtils.equals(replicaSetConfig1.getReplicaSetName(), "test"));
+ Assert.assertTrue(StringUtils.equals(replicaSetConfig1.getHost(), "127.0.0.1:27027"));
+ Assert.assertTrue(replicaSetConfig1.getPosition().getTimeStamp() == 11111111);
+ Assert.assertTrue(replicaSetConfig1.getPosition().getInc() == 111);
+ Assert.assertTrue(replicaSetConfig1.getPosition().isInitSync());
+ }
+
+ private SourceTaskContext emptyTaskContext() {
+ return new SourceTaskContext() {
+ @Override
+ public PositionStorageReader positionStorageReader() {
+ return new PositionStorageReader() {
+ @Override
+ public ByteBuffer getPosition(ByteBuffer partition) {
+ return null;
+ }
+
+ @Override
+ public Map<ByteBuffer, ByteBuffer> getPositions(Collection<ByteBuffer> partitions) {
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public KeyValue configs() {
+ return null;
+ }
+ };
+ }
+
+ @Test
+ public void testContextStart() throws NoSuchFieldException, IllegalAccessException {
+ MongoSourceTask mongoSourceTask = new MongoSourceTask();
+ DefaultKeyValue defaultKeyValue = new DefaultKeyValue();
+ defaultKeyValue.put("mongoAddr", "test/127.0.0.1:27027");
+ defaultKeyValue.put("serverSelectionTimeoutMS", "10");
+
+ Field context = SourceTask.class.getDeclaredField("context");
+ context.setAccessible(true);
+ context.set(mongoSourceTask, TaskContext());
+ mongoSourceTask.start(defaultKeyValue);
+
+ Field replicaSetsContext = MongoSourceTask.class.getDeclaredField("replicaSetsContext");
+ replicaSetsContext.setAccessible(true);
+ ReplicaSetsContext setsContext = (ReplicaSetsContext) replicaSetsContext.get(mongoSourceTask);
+
+ Field replicaSets = ReplicaSetsContext.class.getDeclaredField("replicaSets");
+ replicaSets.setAccessible(true);
+ List<ReplicaSet> replicaSetList = (List<ReplicaSet>) replicaSets.get(setsContext);
+ Assert.assertTrue(replicaSetList.size() == 1);
+ ReplicaSet replicaSet = replicaSetList.get(0);
+ Field replicaSetConfig = ReplicaSet.class.getDeclaredField("replicaSetConfig");
+ replicaSetConfig.setAccessible(true);
+ ReplicaSetConfig replicaSetConfig1 = (ReplicaSetConfig) replicaSetConfig.get(replicaSet);
+ Assert.assertTrue(StringUtils.equals(replicaSetConfig1.getReplicaSetName(), "test"));
+ Assert.assertTrue(StringUtils.equals(replicaSetConfig1.getHost(), "127.0.0.1:27027"));
+ Assert.assertTrue(replicaSetConfig1.getPosition().getTimeStamp() == 22222222);
+ Assert.assertTrue(replicaSetConfig1.getPosition().getInc() == 222);
+ Assert.assertTrue(!replicaSetConfig1.getPosition().isInitSync());
+ }
+
+ private SourceTaskContext TaskContext() {
+ return new SourceTaskContext() {
+ @Override
+ public PositionStorageReader positionStorageReader() {
+ return new PositionStorageReader() {
+ @Override
+ public ByteBuffer getPosition(ByteBuffer partition) {
+
+ Map<String, Object> po = new HashMap<>();
+ po.put("timeStamp", 22222222);
+ po.put("inc", 222);
+ po.put("initSync", false);
+ return ByteBuffer.wrap(JSONObject.toJSONString(po).getBytes());
+ }
+
+ @Override
+ public Map<ByteBuffer, ByteBuffer> getPositions(Collection<ByteBuffer> partitions) {
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public KeyValue configs() {
+ return null;
+ }
+ };
+ }
+}
diff --git a/src/test/java/org/apache/connect/mongo/MongoTest.java b/src/test/java/org/apache/connect/mongo/MongoTest.java
index 7b0291a..98e9a42 100644
--- a/src/test/java/org/apache/connect/mongo/MongoTest.java
+++ b/src/test/java/org/apache/connect/mongo/MongoTest.java
@@ -9,6 +9,14 @@ import com.mongodb.client.MongoCollection;
import io.openmessaging.connector.api.data.EntryType;
import io.openmessaging.connector.api.data.Schema;
import io.openmessaging.connector.api.data.SourceDataEntry;
+import java.lang.reflect.Field;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.connect.mongo.initsync.InitSync;
import org.apache.connect.mongo.replicator.Constants;
import org.apache.connect.mongo.replicator.ReplicaSet;
@@ -23,11 +31,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.lang.reflect.Field;
-import java.nio.ByteBuffer;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicBoolean;
-
public class MongoTest {
private MongoClient mongoClient;
@@ -35,11 +38,10 @@ public class MongoTest {
@Before
public void before() {
MongoClientSettings.Builder builder = MongoClientSettings.builder();
- builder.applyConnectionString(new ConnectionString("mongodb://127.0.0.1:27077"));
+ builder.applyConnectionString(new ConnectionString("mongodb://127.0.0.1:27027"));
mongoClient = MongoClients.create(builder.build());
}
-
@Test
public void testConvertEvent() {
Document oplog = new Document();
@@ -60,10 +62,8 @@ public class MongoTest {
Assert.assertEquals(document, event.getEventData().get());
Assert.assertEquals("testR", event.getReplicaSetName());
-
}
-
@Test
public void testInitSyncCopy() throws NoSuchFieldException, IllegalAccessException, InterruptedException {
MongoCollection<Document> collection = mongoClient.getDatabase("test").getCollection("person");
diff --git a/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java b/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
index 8613c42..16cb959 100644
--- a/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
+++ b/src/test/java/org/apache/connect/mongo/ReplicaContextTest.java
@@ -20,10 +20,9 @@ public class ReplicaContextTest {
context = new ReplicaSetsContext(sourceTaskConfig);
}
-
@Test
public void testCreateMongoClient() {
- MongoClient mongoClient = context.createMongoClient(new ReplicaSetConfig("shardName1", "rep1", "127.0.0.1:27017"));
+ MongoClient mongoClient = context.createMongoClient(new ReplicaSetConfig("shardName1", "", "127.0.0.1:27027"));
MongoIterable<String> collectionNames = mongoClient.getDatabase("local").listCollectionNames();
MongoCursor<String> iterator = collectionNames.iterator();
while (iterator.hasNext()) {
diff --git a/src/test/java/org/apache/connect/mongo/ReplicaSetTest.java b/src/test/java/org/apache/connect/mongo/ReplicaSetTest.java
new file mode 100644
index 0000000..07eefae
--- /dev/null
+++ b/src/test/java/org/apache/connect/mongo/ReplicaSetTest.java
@@ -0,0 +1,59 @@
+package org.apache.connect.mongo;
+
+import java.lang.reflect.Field;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.connect.mongo.replicator.ReplicaSet;
+import org.apache.connect.mongo.replicator.ReplicaSetConfig;
+import org.apache.connect.mongo.replicator.ReplicaSetsContext;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class ReplicaSetTest {
+
+ private ReplicaSet replicaSet;
+
+ private SourceTaskConfig sourceTaskConfig;
+
+ private ReplicaSetConfig replicaSetConfig;
+
+ private ReplicaSetsContext replicaSetsContext;
+
+ @Before
+ public void before() {
+ this.sourceTaskConfig = new SourceTaskConfig();
+ this.replicaSetConfig = new ReplicaSetConfig("shardName1", "", "127.0.0.1:27027");
+ this.replicaSetsContext = new ReplicaSetsContext(sourceTaskConfig);
+ this.replicaSet = new ReplicaSet(replicaSetConfig, replicaSetsContext);
+ }
+
+ @Test
+ public void testStartAndShutDown() throws NoSuchFieldException, IllegalAccessException {
+ replicaSet.start();
+ Field field = ReplicaSet.class.getDeclaredField("running");
+ field.setAccessible(true);
+ AtomicBoolean o = (AtomicBoolean) field.get(replicaSet);
+ Assert.assertTrue(o.get());
+ replicaSet.shutdown();
+ Assert.assertFalse(o.get());
+ }
+
+ @Test
+ public void testPause() throws Exception {
+ replicaSet.pause();
+ Field field = ReplicaSet.class.getDeclaredField("pause");
+ field.setAccessible(true);
+ boolean pause = (boolean) field.get(replicaSet);
+ Assert.assertTrue(pause);
+ }
+
+ @Test
+ public void testResume() throws Exception {
+ replicaSet.resume();
+ Field field = ReplicaSet.class.getDeclaredField("pause");
+ field.setAccessible(true);
+ boolean pause = (boolean) field.get(replicaSet);
+ Assert.assertFalse(pause);
+ }
+
+}
diff --git a/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java b/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java
index e69eac6..5276f4f 100644
--- a/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java
+++ b/src/test/java/org/apache/connect/mongo/ReplicaSetsTest.java
@@ -1,29 +1,25 @@
package org.apache.connect.mongo;
+import java.util.Map;
import org.apache.connect.mongo.replicator.ReplicaSetConfig;
import org.apache.connect.mongo.replicator.ReplicaSets;
import org.junit.Assert;
import org.junit.Test;
-import java.util.Map;
-
public class ReplicaSetsTest {
-
@Test(expected = IllegalArgumentException.class)
- public void testCreatReplicaSetsException01() {
+ public void testCreatReplicaSetsExceptionWithoutMongoAddr() {
ReplicaSets.create("");
}
-
@Test(expected = IllegalArgumentException.class)
- public void testCreatReplicaSetsException02() {
+ public void testCreatReplicaSetsExceptioWithoutReplicaSetName() {
ReplicaSets.create("127.0.0.1:27081");
}
-
@Test
- public void testCreatReplicaSets01() {
+ public void testCreatReplicaSetsSpecialReplicaSetName() {
ReplicaSets replicaSets = ReplicaSets.create("replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083");
Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
Assert.assertTrue(replicaSetConfigMap.size() == 1);
@@ -32,9 +28,8 @@ public class ReplicaSetsTest {
Assert.assertEquals("replicaName1", replicaSetConfigMap.get("replicaName1").getReplicaSetName());
}
-
@Test
- public void testCreatReplicaSets02() {
+ public void testCreatReplicaSetsSpecialShardNameAndReplicaSetName() {
ReplicaSets replicaSets = ReplicaSets.create("shardName1=replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083");
Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
Assert.assertTrue(replicaSetConfigMap.size() == 1);
@@ -44,9 +39,8 @@ public class ReplicaSetsTest {
Assert.assertEquals("shardName1", replicaSetConfigMap.get("replicaName1").getShardName());
}
-
@Test
- public void testCreatReplicaSets03() {
+ public void testCreatReplicaSetsMutiMongoAddr() {
ReplicaSets replicaSets = ReplicaSets.create("shardName1=replicaName1/127.0.0.1:27081,127.0.0.1:27082,127.0.0.1:27083;shardName2=replicaName2/127.0.0.1:27281,127.0.0.1:27282,127.0.0.1:27283");
Map<String, ReplicaSetConfig> replicaSetConfigMap = replicaSets.getReplicaConfigByName();
Assert.assertTrue(replicaSetConfigMap.size() == 2);
@@ -55,7 +49,6 @@ public class ReplicaSetsTest {
Assert.assertEquals("replicaName1", replicaSetConfigMap.get("replicaName1").getReplicaSetName());
Assert.assertEquals("shardName1", replicaSetConfigMap.get("replicaName1").getShardName());
-
Assert.assertNotNull(replicaSetConfigMap.get("replicaName2"));
Assert.assertEquals("127.0.0.1:27281,127.0.0.1:27282,127.0.0.1:27283", replicaSetConfigMap.get("replicaName2").getHost());
Assert.assertEquals("replicaName2", replicaSetConfigMap.get("replicaName2").getReplicaSetName());
[rocketmq-connect] 05/13: fix some bug
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 70caed3673b4d5b783447622410ddf3dcf9bdfee
Author: 李平 <lp...@alibaba-inc.com>
AuthorDate: Thu Aug 8 16:25:52 2019 +0800
fix some bug
---
.../org/apache/connect/mongo/MongoReplicatorConfig.java | 13 ++++++++++---
.../org/apache/connect/mongo/replicator/ReplicatorTask.java | 4 ++--
2 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java b/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
index 9f17aab..b7044ec 100644
--- a/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
+++ b/src/main/java/org/apache/connect/mongo/MongoReplicatorConfig.java
@@ -2,6 +2,7 @@ package org.apache.connect.mongo;
import io.openmessaging.KeyValue;
import org.apache.commons.lang3.StringUtils;
+import org.bson.BsonTimestamp;
import java.lang.reflect.Method;
import java.util.HashSet;
@@ -14,7 +15,7 @@ public class MongoReplicatorConfig {
private String mongoUserName;
private String mongoPassWord;
private String interestDbAndCollection;
- private long positionTimeStamp;
+ private int positionTimeStamp;
private int positionInc;
private String dataSync;
private int copyThread = Runtime.getRuntime().availableProcessors();
@@ -42,11 +43,11 @@ public class MongoReplicatorConfig {
this.copyThread = copyThread;
}
- public long getPositionTimeStamp() {
+ public int getPositionTimeStamp() {
return positionTimeStamp;
}
- public void setPositionTimeStamp(long positionTimeStamp) {
+ public void setPositionTimeStamp(int positionTimeStamp) {
this.positionTimeStamp = positionTimeStamp;
}
@@ -153,4 +154,10 @@ public class MongoReplicatorConfig {
}
return replicaSet + ":" + mongoAddr;
}
+
+
+ public BsonTimestamp getPosition() {
+ return new BsonTimestamp(positionTimeStamp, positionInc);
+ }
+
}
diff --git a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
index 9e4c67b..32bb25b 100644
--- a/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
+++ b/src/main/java/org/apache/connect/mongo/replicator/ReplicatorTask.java
@@ -7,9 +7,9 @@ import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import org.apache.connect.mongo.MongoReplicatorConfig;
+import org.apache.connect.mongo.initsync.InitSync;
import org.apache.connect.mongo.replicator.event.DocumentConvertEvent;
import org.apache.connect.mongo.replicator.event.ReplicationEvent;
-import org.apache.connect.mongo.initsync.InitSync;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,7 +46,7 @@ public class ReplicatorTask implements Runnable {
FindIterable<Document> iterable;
if (mongoReplicatorConfig.getPositionTimeStamp() > 0 && mongoReplicatorConfig.getPositionTimeStamp() < System.currentTimeMillis()) {
iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find(
- Filters.gt("ts", mongoReplicatorConfig.getPositionTimeStamp()));
+ Filters.gt("ts", mongoReplicatorConfig.getPosition()));
} else {
iterable = localDataBase.getCollection(Constants.MONGO_OPLOG_RS).find();
}