You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rocketmq.apache.org by zh...@apache.org on 2022/03/02 03:20:08 UTC
[rocketmq-connect] branch master updated (386adcc -> 24c083e)
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git.
from 386adcc Add 'connector/rocketmq-connect-hudi/' from commit '5da4b78705108ac6d260283cd38f9be08d2590b9'
new a9fb3af Init rocketmq-connect-jdbc
new 13b6ae3 Add Jdbc Source Connector and Do Unit Test
new 9716b10 Update pom.xml
new 5db4b46 Update Config.java
new 675099d Update JdbcSourceConnector.java
new 3c5f659 Add Jdbc Source Task (To be continued)
new 2cc48d9 Add Config File
new 5a83890 Add JdbcSourceTask
new e915f67 Add JdbcSourceTask and Schema
new 9ea583f Add SourceJdbcTask and Schema
new 8ecde45 Add JdbcSourceTask and Schema
new a1d1200 Delete ReplicatorTest.java
new b1acd43 delete lib
new 488223c develop the jdbcsource connector
new d0569b7 Update Querier.java
new cd6410d Update JdbcSourceConnectorTest.java
new 1376d82 Develop TimestampIncrementingQuerier Mode
new 4849ff5 Update TimestampIncrementingQuerier.java
new e72688b Update Querier.java
new 0521831 Delete JdbcSourceTaskTest.java
new 8a266e8 update readme
new 717d4a2 Update Schema.java
new 5457c06 Update JdbcSourceTask.java
new db7644a Update JdbcSourceTask.java
new 182fd46 [ISSUE #441] Add Jdbc Sink Connector (#442)
new 42cbfb3 [ISSUE #485] Support repeat consumption (#486)
new d06e10c [ISSUE #487] Jdbc source connector support syncing data with white ta… (#488)
new 759e422 [ISSUE #498] update rocketmq-jdbc-connector README.md (#499)
new 341d6f3 [ISSUE #489] JDBC Connector support divide task by topic strategy (#490)
new 252e26a [ISSUE #495] jdbc-sink-connector support divide task by queue (#496)
new 7c81b41 Update connector dependency to the latest version
new 7ebfedf fix(jdbc-connect) removed unused class
new b9802ae fix(jdbc-connect) removed unused class (#544)
new e325d72 1.add rocketmq-tools dependency so rocketmq-connect-jdbc can run on it (#537)
new 5420e44 1.add required fields in RockeMQ jdbc connector 2. add docs (#539)
new 0b56e47 [ISSUE #545]bug fix (#546)
new a1f2ff2 [ISSUE #550] Removed unnecessary value of REQUEST_CONFIG in Config.java (#551)
new 9849db6 [ISSUE #554] Update druid version and set ConnectionErrorRetryAttempts
new 1a49e60 [ISSUE #558] An ugly solution for fetch topic list error
new b425260 [ISSUE #570] ASoC connect runtime optimization: CLI (#622)
new 5f40a88 Update README.md (#553)
new 6708ada Merge branch 'master' of github.com:apache/rocketmq-externals
new 24c083e Add 'connector/rocketmq-connect-jdbc/' from commit '6708ada617d9f6cfef5ca42a3c2f97af44603a89'
The 43 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
connector/rocketmq-connect-jdbc/README.md | 85 ++++++
.../pom.xml | 147 +++++-----
.../rocketmq/connect/jdbc}/common/CloneUtils.java | 18 +-
.../rocketmq/connect/jdbc}/common/ConstDefine.java | 6 +-
.../rocketmq/connect/jdbc/common/DBUtils.java | 212 ++++++++++++++
.../rocketmq/connect/jdbc/common}/Utils.java | 15 +-
.../rocketmq/connect/jdbc}/config/Config.java | 181 ++++++++----
.../rocketmq/connect/jdbc}/config/ConfigUtil.java | 20 +-
.../rocketmq/connect/jdbc/config}/DataType.java | 2 +-
.../connect/jdbc}/config/DbConnectorConfig.java | 32 +--
.../jdbc}/config/SinkDbConnectorConfig.java | 36 +--
.../jdbc}/config/SourceDbConnectorConfig.java | 32 +--
.../connect/jdbc}/config/TaskDivideConfig.java | 17 +-
.../connect/jdbc}/config/TaskTopicInfo.java | 15 +-
.../connect/jdbc/connector/JdbcSinkConnector.java} | 89 +++---
.../connect/jdbc/connector/JdbcSinkTask.java} | 79 ++----
.../jdbc/connector/JdbcSourceConnector.java} | 29 +-
.../connect/jdbc/connector/JdbcSourceTask.java} | 85 +++---
.../rocketmq/connect/jdbc/schema/Database.java | 109 ++++++++
.../rocketmq/connect/jdbc/schema/Schema.java | 123 ++++++++
.../rocketmq/connect/jdbc}/schema/Table.java | 6 +-
.../jdbc}/schema/column/BigIntColumnParser.java | 2 +-
.../connect/jdbc}/schema/column/ColumnParser.java | 18 +-
.../jdbc}/schema/column/DateTimeColumnParser.java | 2 +-
.../jdbc}/schema/column/DefaultColumnParser.java | 2 +-
.../jdbc}/schema/column/EnumColumnParser.java | 2 +-
.../jdbc}/schema/column/IntColumnParser.java | 2 +-
.../jdbc}/schema/column/SetColumnParser.java | 2 +-
.../jdbc}/schema/column/StringColumnParser.java | 2 +-
.../jdbc}/schema/column/TimeColumnParser.java | 2 +-
.../jdbc}/schema/column/YearColumnParser.java | 2 +-
.../apache/rocketmq/connect/jdbc/sink/Updater.java | 258 +++++++++++++++++
.../rocketmq/connect/jdbc/source/Querier.java | 173 ++++++++++++
.../jdbc/source/TimestampIncrementingQuerier.java | 311 +++++++++++++++++++++
.../connect/jdbc}/strategy/DivideStrategyEnum.java | 2 +-
.../connect/jdbc/strategy/DivideTaskByQueue.java | 72 +++++
.../connect/jdbc}/strategy/DivideTaskByTopic.java | 63 +++--
.../connect/jdbc}/strategy/TaskDivideStrategy.java | 12 +-
.../jdbc/connector/JdbcSourceConnectorTest.java} | 31 +-
39 files changed, 1790 insertions(+), 506 deletions(-)
create mode 100644 connector/rocketmq-connect-jdbc/README.md
copy connector/{rocketmq-connect-cassandra => rocketmq-connect-jdbc}/pom.xml (73%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/common/CloneUtils.java (50%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/common/ConstDefine.java (80%)
create mode 100644 connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
copy connector/{rocketmq-connect-hudi/src/main/java/org/apache/rocketmq/connect/hudi/config => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common}/Utils.java (87%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/config/Config.java (70%)
copy connector/{rocketmq-connect-hudi/src/main/java/org/apache/rocketmq/connect/hudi => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/config/ConfigUtil.java (71%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/common => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config}/DataType.java (94%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/config/DbConnectorConfig.java (60%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/config/SinkDbConnectorConfig.java (67%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/config/SourceDbConnectorConfig.java (65%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/config/TaskDivideConfig.java (84%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/config/TaskTopicInfo.java (72%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/connector/CassandraSinkConnector.java => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java} (73%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/connector/CassandraSinkTask.java => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java} (55%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/connector/CassandraSourceConnector.java => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java} (77%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/connector/CassandraSourceTask.java => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java} (75%)
create mode 100644 connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
create mode 100644 connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/Table.java (95%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/BigIntColumnParser.java (96%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/ColumnParser.java (90%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/DateTimeColumnParser.java (96%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/DefaultColumnParser.java (95%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/EnumColumnParser.java (95%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/IntColumnParser.java (96%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/SetColumnParser.java (96%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/StringColumnParser.java (96%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/TimeColumnParser.java (95%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/schema/column/YearColumnParser.java (95%)
create mode 100644 connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
create mode 100644 connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
create mode 100644 connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/strategy/DivideStrategyEnum.java (93%)
create mode 100644 connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/strategy/DivideTaskByTopic.java (85%)
copy connector/{rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra => rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc}/strategy/TaskDivideStrategy.java (80%)
copy connector/{rocketmq-connect-activemq/src/test/java/org/apache/rocketmq/connect/activemq/connector/ActivemqConnectorTest.java => rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java} (73%)
[rocketmq-connect] 27/43: [ISSUE #487] Jdbc source connector support syncing data with white ta… (#488)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit d06e10c29832f886ca15ecd83a4ca65bfd6d4d85
Author: lrhkobe <34...@users.noreply.github.com>
AuthorDate: Thu Dec 19 09:22:47 2019 +0800
[ISSUE #487] Jdbc source connector support syncing data with white ta… (#488)
* [ISSUE #487] Jdbc source connector support syncing data with white table column value
* [ISSUE #487] Jdbc source connector support syncing data with white table column value
---
.../rocketmq/connect/jdbc/schema/Database.java | 6 ++-
.../rocketmq/connect/jdbc/schema/Schema.java | 16 +++----
.../apache/rocketmq/connect/jdbc/schema/Table.java | 11 +++++
.../rocketmq/connect/jdbc/source/Querier.java | 56 +++++++++++++++-------
4 files changed, 63 insertions(+), 26 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
index 49e28cd..33a9a22 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
@@ -40,9 +40,12 @@ public class Database {
public Set<String> tableWhiteList;
- public Database(String name, Connection connection, Set<String> tableWhiteList) {
+ public Map<String, Map<String, String>> tableFilterMap;
+
+ public Database(String name, Connection connection, Set<String> tableWhiteList, Map<String, Map<String, String>> tableFilterMap) {
this.name = name;
this.connection = connection;
+ this.tableFilterMap = tableFilterMap;
this.tableWhiteList = tableWhiteList;
}
@@ -73,6 +76,7 @@ public class Database {
table.addCol(colName);
table.addParser(columnParser);
table.addRawDataType(dataType);
+ table.setFilterMap(tableFilterMap.get(tableName));
}
} finally {
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
index 16d636f..1cfaf2c 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
@@ -34,18 +34,18 @@ public class Schema {
Arrays.asList(new String[] {"information_schema", "mysql", "performance_schema", "sys"})
);
- public Set<String> dataBaseWhiteList;
-
- public Set<String> tableWhiteList;
-
private Connection connection;
private Map<String, Database> dbMap;
+ public Map<String, Set<String>> dbTableMap;
+
+ public Map<String, Map<String, String>> tableFilterMap;
+
public Schema(Connection connection) {
this.connection = connection;
- this.dataBaseWhiteList = new HashSet<>();
- this.tableWhiteList = new HashSet<>();
+ this.dbTableMap = new HashMap<>();
+ this.tableFilterMap = new HashMap<>();
}
public void load() throws SQLException {
@@ -61,8 +61,8 @@ public class Schema {
while (rs.next()) {
String dbName = rs.getString(1);
- if (!IGNORED_DATABASES.contains(dbName) && dataBaseWhiteList.contains(dbName)) {
- Database database = new Database(dbName, connection, tableWhiteList);
+ if (!IGNORED_DATABASES.contains(dbName) && dbTableMap.keySet().contains(dbName)) {
+ Database database = new Database(dbName, connection, dbTableMap.get(dbName), tableFilterMap);
dbMap.put(dbName, database);
}
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
index 8c9a42d..891fb9a 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
@@ -18,8 +18,11 @@
package org.apache.rocketmq.connect.jdbc.schema;
import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
+
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
public class Table {
@@ -29,6 +32,7 @@ public class Table {
private List<ColumnParser> parserList = new LinkedList<>();
private List<String> rawDataTypeList = new LinkedList<>();
private List<Object> dataList = new LinkedList<>();
+ private Map<String, String> filterMap = new HashMap<>();
public Table(String database, String table) {
this.database = database;
@@ -87,4 +91,11 @@ public class Table {
this.colList = colList;
}
+ public Map<String, String> getFilterMap() {
+ return filterMap;
+ }
+
+ public void setFilterMap(Map<String, String> filterMap) {
+ this.filterMap = filterMap;
+ }
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index 8da3f21..d2544f9 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -1,5 +1,6 @@
package org.apache.rocketmq.connect.jdbc.source;
+import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.rocketmq.connect.jdbc.config.Config;
import org.apache.rocketmq.connect.jdbc.schema.Database;
@@ -91,7 +92,7 @@ public class Querier {
public void poll() {
try {
PreparedStatement stmt;
- String query = "select * from ";
+ StringBuilder query = new StringBuilder("select * from ");
LinkedList<Table> tableLinkedList = new LinkedList<>();
for (Map.Entry<String, Database> entry : schema.getDbMap().entrySet()) {
String db = entry.getKey();
@@ -99,19 +100,35 @@ public class Querier {
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
String tb = tableEntry.getKey();
+ query.append(db + "." + tb);
+ Table t = tableEntry.getValue();
+ Map<String, String> tableFilterMap = t.getFilterMap();
+ if (tableFilterMap != null && !tableFilterMap.keySet().contains("NO-FILTER")){
+ query = query.append(" where ");
+ int count = 0;
+ for (String key : tableFilterMap.keySet()){
+ count ++;
+ String value = tableFilterMap.get(key);
+ if (count != 1){
+ query.append(" and ");
+ }
+ String condition = key + "=" + "'" + value + "'";
+ query.append(condition);
+ }
+ }
stmt = connection.prepareStatement(query + db + "." + tb);
ResultSet rs;
rs = stmt.executeQuery();
List<String> colList = tableEntry.getValue().getColList();
- List<String> DataTypeList = tableEntry.getValue().getRawDataTypeList();
- List<ColumnParser> ParserList = tableEntry.getValue().getParserList();
+ List<String> dataTypeList = tableEntry.getValue().getRawDataTypeList();
+ List<ColumnParser> parserList = tableEntry.getValue().getParserList();
while (rs.next()) {
Table table = new Table(db, tb);
//System.out.print("|");
table.setColList(colList);
- table.setRawDataTypeList(DataTypeList);
- table.setParserList(ParserList);
+ table.setRawDataTypeList(dataTypeList);
+ table.setParserList(parserList);
for (String string : colList) {
table.getDataList().add(rs.getObject(string));
@@ -132,18 +149,23 @@ public class Querier {
public void start() throws Exception {
String whiteDataBases = config.getWhiteDataBase();
- String whiteTables = config.getWhiteTable();
-
- if (!StringUtils.isEmpty(whiteDataBases)) {
- Arrays.asList(whiteDataBases.trim().split(",")).forEach(whiteDataBase -> {
- Collections.addAll(schema.dataBaseWhiteList, whiteDataBase);
- });
- }
-
- if (!StringUtils.isEmpty(whiteTables)) {
- Arrays.asList(whiteTables.trim().split(",")).forEach(whiteTable -> {
- Collections.addAll(schema.tableWhiteList, whiteTable);
- });
+ JSONObject whiteDataBaseObject = JSONObject.parseObject(whiteDataBases);
+
+ if (whiteDataBaseObject != null){
+ for (String whiteDataBaseName : whiteDataBaseObject.keySet()){
+ JSONObject whiteTableObject = (JSONObject)whiteDataBaseObject.get(whiteDataBaseName);
+ HashSet<String> whiteTableSet = new HashSet<>();
+ for (String whiteTableName : whiteTableObject.keySet()){
+ Collections.addAll(whiteTableSet, whiteTableName);
+ HashMap<String, String> filterMap = new HashMap<>();
+ JSONObject tableFilterObject = (JSONObject)whiteTableObject.get(whiteTableName);
+ for(String filterKey : tableFilterObject.keySet()){
+ filterMap.put(filterKey, tableFilterObject.getString(filterKey));
+ }
+ schema.tableFilterMap.put(whiteTableName, filterMap);
+ }
+ schema.dbTableMap.put(whiteDataBaseName, whiteTableSet);
+ }
}
schema.load();
log.info("load schema success");
[rocketmq-connect] 24/43: Update JdbcSourceTask.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit db7644ae06f9edef81098c699cbd070a7989b282
Author: Yuchen Li <yu...@126.com>
AuthorDate: Mon Sep 9 19:45:33 2019 +0800
Update JdbcSourceTask.java
---
.../apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index e01c226..767c3aa 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -118,12 +118,9 @@ public class JdbcSourceTask extends SourceTask {
@Override
public void start(KeyValue props) {
- try {
- config = new Config();
- config.load(props);
- } catch (Exception e) {
- log.error("Cannot start Jdbc Source Task because of configuration error{}", e);
- }
+ config = new Config();
+ config.load(props);
+
Map<Map<String, String>, Map<String, Object>> offsets = null;
String mode = config.mode;
if (mode.equals("bulk")) {
[rocketmq-connect] 17/43: Develop TimestampIncrementingQuerier Mode
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 1376d8206633c494dab351b0b88723be0a6af102
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Thu Aug 15 21:57:54 2019 +0800
Develop TimestampIncrementingQuerier Mode
---
.../org/apache/rocketmq/connect/jdbc/Config.java | 7 +-
.../jdbc/connector/JdbcSourceConnector.java | 18 +-
.../connect/jdbc/connector/JdbcSourceTask.java | 100 +++++--
.../rocketmq/connect/jdbc/source/JdbcUtils.java | 198 +++++++++++++
.../rocketmq/connect/jdbc/source/Querier.java | 77 +++--
.../jdbc/source/TimestampIncrementingQuerier.java | 315 +++++++++++++++++++++
.../connect/jdbc/connector/JdbcSourceTaskTest.java | 89 +++++-
7 files changed, 727 insertions(+), 77 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
index 533d53b..f93c4db 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
@@ -27,8 +27,6 @@ import java.util.List;
import java.util.Set;
public class Config {
- @SuppressWarnings("serial")
-
private static final Logger LOG = LoggerFactory.getLogger(Config.class);
/* Database Connection Config */
@@ -68,11 +66,12 @@ public class Config {
add("jdbcUrl");
add("jdbcUsername");
add("jdbcPassword");
- // add("mode");
- // add("rocketmqTopic");
+ add("mode");
+ add("rocketmqTopic");
}
};
+
public void load(KeyValue props) {
log.info("Config.load.start");
properties2Object(props, this);
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
index bdbeb8b..4a870c0 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
@@ -19,17 +19,15 @@ package org.apache.rocketmq.connect.jdbc.connector;
import java.util.ArrayList;
import java.util.List;
-import java.util.Set;
-
-import io.openmessaging.KeyValue;
-import io.openmessaging.connector.api.Task;
-import io.openmessaging.connector.api.source.SourceConnector;
import org.apache.rocketmq.connect.jdbc.Config;
-import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import io.openmessaging.connector.api.source.SourceConnector;
+
public class JdbcSourceConnector extends SourceConnector {
private static final Logger log = LoggerFactory.getLogger(JdbcSourceConnector.class);
private KeyValue config;
@@ -37,7 +35,7 @@ public class JdbcSourceConnector extends SourceConnector {
@Override
public String verifyAndSetConfig(KeyValue config) {
- log.info("JdbcSourceConnector verifyAndSetConfig enter");
+ log.info("1216123 JdbcSourceConnector verifyAndSetConfig enter");
for (String requestKey : Config.REQUEST_CONFIG) {
if (!config.containsKey(requestKey)) {
@@ -59,11 +57,13 @@ public class JdbcSourceConnector extends SourceConnector {
}
- @Override public void pause() {
+ @Override
+ public void pause() {
}
- @Override public void resume() {
+ @Override
+ public void resume() {
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index 91659ec..45252bb 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -1,5 +1,4 @@
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -20,20 +19,31 @@
package org.apache.rocketmq.connect.jdbc.connector;
import io.openmessaging.connector.api.source.SourceTask;
+
import java.nio.ByteBuffer;
+import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
import org.apache.rocketmq.connect.jdbc.Config;
import org.apache.rocketmq.connect.jdbc.schema.Table;
import org.apache.rocketmq.connect.jdbc.source.Querier;
+import org.apache.rocketmq.connect.jdbc.source.TimestampIncrementingQuerier;
import org.apache.rocketmq.connect.jdbc.schema.column.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSON;
+
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.data.EntryType;
import io.openmessaging.connector.api.data.Schema;
@@ -49,26 +59,31 @@ public class JdbcSourceTask extends SourceTask {
private Config config;
- private List<Table> list = new LinkedList<>();
-
- Querier querier = new Querier();
+ BlockingQueue<Querier> tableQueue = new LinkedBlockingQueue<Querier>();
+ static final String INCREMENTING_FIELD = "incrementing";
+ static final String TIMESTAMP_FIELD = "timestamp";
+ private Querier querier;
@Override
public Collection<SourceDataEntry> poll() {
List<SourceDataEntry> res = new ArrayList<>();
try {
-
- JSONObject jsonObject = new JSONObject();
- jsonObject.put("nextQuery", "database");
- jsonObject.put("nextPosition", "10");
- //To be Continued
+ if (tableQueue.size() > 1)
+ querier = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
+ else
+ querier = tableQueue.peek();
+ Timer timer = new java.util.Timer();
+ try {
+ Thread.currentThread();
+ Thread.sleep(1000);//毫秒
+ } catch (Exception e) {
+ throw e;
+ }
querier.poll();
- log.info("querier.poll, start");
- int mm = 0;
for (Table dataRow : querier.getList()) {
- System.out.println(dataRow.getColList().get(0));
- log.info("xunhuankaishi");
- log.info("Received {} record: {} ", dataRow.getColList().get(0), mm++);
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put("nextQuery", "database");
+ jsonObject.put("nextPosition", "table");
Schema schema = new Schema();
schema.setDataSource(dataRow.getDatabase());
schema.setName(dataRow.getName());
@@ -80,37 +95,60 @@ public class JdbcSourceTask extends SourceTask {
schema.getFields().add(field);
}
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
- dataEntryBuilder.timestamp(System.currentTimeMillis())
- .queue(dataRow.getName())
- .entryType(EntryType.CREATE);
+ dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(dataRow.getName())
+ .entryType(EntryType.CREATE);
for (int i = 0; i < dataRow.getColList().size(); i++) {
Object value = dataRow.getDataList().get(i);
- System.out.println(1);
- System.out.println(dataRow.getColList().get(i) + "|" + value);
- dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSON.toJSONString(value));
+ // System.out.println(dataRow.getColList().get(i) + "|" + value);
+ dataEntryBuilder.putFiled(dataRow.getColList().get(i), value);
}
+
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap(config.jdbcUrl.getBytes("UTF-8")),
- ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8")));
+ ByteBuffer.wrap(config.jdbcUrl.getBytes("UTF-8")),
+ ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8")));
res.add(sourceDataEntry);
+
}
} catch (Exception e) {
log.error("JDBC task poll error, current config:" + JSON.toJSONString(config), e);
}
+ log.info("dataEntry poll successfully,{}", res);
return res;
}
@Override
public void start(KeyValue props) {
try {
- this.config = new Config();
- this.config.load(props);
- log.info("querier.start");
- querier.start();
-
+ config = new Config();
+ config.load(props);
} catch (Exception e) {
- log.error("JDBC task start failed.", e);
+ log.error("Cannot start Jdbc Source Task because of configuration error{}", e);
+ }
+ Map<Map<String, String>, Map<String, Object>> offsets = null;
+ String mode = config.mode;
+ if (mode.equals("bulk")) {
+ Querier querier = new Querier();
+ try {
+ querier.setConfig(config);
+ querier.start();
+ tableQueue.add(querier);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ } else {
+ TimestampIncrementingQuerier querier = new TimestampIncrementingQuerier();
+ try {
+ querier.setConfig(config);
+ querier.start();
+ tableQueue.add(querier);
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
}
+
}
@Override
@@ -118,11 +156,13 @@ public class JdbcSourceTask extends SourceTask {
querier.stop();
}
- @Override public void pause() {
+ @Override
+ public void pause() {
}
- @Override public void resume() {
+ @Override
+ public void resume() {
}
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcUtils.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcUtils.java
new file mode 100644
index 0000000..cbcca6a
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcUtils.java
@@ -0,0 +1,198 @@
+
+/**
+ * Copyright 2015 Confluent Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.apache.rocketmq.connect.jdbc.source;
+import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.TimeZone;
+
+/**
+ * Utilties for interacting with a JDBC database.
+ */
+public class JdbcUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(JdbcSourceTask.class);
+
+ /**
+ * The default table types to include when listing tables if none are specified. Valid values
+ * are those specified by the @{java.sql.DatabaseMetaData#getTables} method's TABLE_TYPE column.
+ * The default only includes standard, user-defined tables.
+ */
+ public static final Set<String> DEFAULT_TABLE_TYPES = Collections.unmodifiableSet(
+ new HashSet<String>(Arrays.asList("TABLE"))
+ );
+
+ private static final int GET_TABLES_TYPE_COLUMN = 4;
+ private static final int GET_TABLES_NAME_COLUMN = 3;
+
+ private static final int GET_COLUMNS_COLUMN_NAME = 4;
+ private static final int GET_COLUMNS_IS_NULLABLE = 18;
+ private static final int GET_COLUMNS_IS_AUTOINCREMENT = 23;
+
+
+ private static ThreadLocal<SimpleDateFormat> DATE_FORMATTER = new ThreadLocal<SimpleDateFormat>() {
+ @Override
+ protected SimpleDateFormat initialValue() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return sdf;
+ }
+ };
+
+ /**
+ * Get a list of tables in the database. This uses the default filters, which only include
+ * user-defined tables.
+ * @param conn database connection
+ * @return a list of tables
+ * @throws SQLException
+ */
+ public static List<String> getTables(Connection conn) throws SQLException {
+ return getTables(conn, DEFAULT_TABLE_TYPES);
+ }
+
+ /**
+ * Get a list of table names in the database.
+ * @param conn database connection
+ * @param types a set of table types that should be included in the results
+ * @throws SQLException
+ */
+ public static List<String> getTables(Connection conn, Set<String> types) throws SQLException {
+ DatabaseMetaData metadata = conn.getMetaData();
+ ResultSet rs = metadata.getTables(null, null, "%", null);
+ List<String> tableNames = new ArrayList<String>();
+ while (rs.next()) {
+ if (types.contains(rs.getString(GET_TABLES_TYPE_COLUMN))) {
+ String colName = rs.getString(GET_TABLES_NAME_COLUMN);
+ // SQLite JDBC driver does not correctly mark these as system tables
+ if (metadata.getDatabaseProductName().equals("SQLite") && colName.startsWith("sqlite_")) {
+ continue;
+ }
+
+ tableNames.add(colName);
+ }
+ }
+ return tableNames;
+ }
+
+ /**
+ * Look up the autoincrement column for the specified table.
+ * @param conn database connection
+ * @param table the table to
+ * @return the name of the column that is an autoincrement column, or null if there is no
+ * autoincrement column or more than one exists
+ * @throws SQLException
+ */
+ public static String getAutoincrementColumn(Connection conn, String table) throws SQLException {
+ String result = null;
+ int matches = 0;
+
+ ResultSet rs = conn.getMetaData().getColumns(null, null, table, "%");
+ // Some database drivers (SQLite) don't include all the columns
+ if (rs.getMetaData().getColumnCount() >= GET_COLUMNS_IS_AUTOINCREMENT) {
+ while(rs.next()) {
+ if (rs.getString(GET_COLUMNS_IS_AUTOINCREMENT).equals("YES")) {
+ result = rs.getString(GET_COLUMNS_COLUMN_NAME);
+ matches++;
+ }
+ }
+ return (matches == 1 ? result : null);
+ }
+
+ // Fallback approach is to query for a single row. This unfortunately does not work with any
+ // empty table
+ log.trace("Falling back to SELECT detection of auto-increment column for {}:{}", conn, table);
+ Statement stmt = conn.createStatement();
+ try {
+ String quoteString = getIdentifierQuoteString(conn);
+ rs = stmt.executeQuery("SELECT * FROM " + quoteString + table + quoteString + " LIMIT 1");
+ ResultSetMetaData rsmd = rs.getMetaData();
+ for(int i = 1; i < rsmd.getColumnCount(); i++) {
+ if (rsmd.isAutoIncrement(i)) {
+ result = rsmd.getColumnName(i);
+ matches++;
+ }
+ }
+ } finally {
+ rs.close();
+ stmt.close();
+ }
+ return (matches == 1 ? result : null);
+ }
+
+ public static boolean isColumnNullable(Connection conn, String table, String column)
+ throws SQLException {
+ ResultSet rs = conn.getMetaData().getColumns(null, null, table, column);
+ if (rs.getMetaData().getColumnCount() > GET_COLUMNS_IS_NULLABLE) {
+ // Should only be one match
+ if (!rs.next()) {
+ return false;
+ }
+ String val = rs.getString(GET_COLUMNS_IS_NULLABLE);
+ return rs.getString(GET_COLUMNS_IS_NULLABLE).equals("YES");
+ }
+
+ return false;
+ }
+
+ /**
+ * Format the given Date assuming UTC timezone in a format supported by SQL.
+ * @param date the date to convert to a String
+ * @return the formatted string
+ */
+ public static String formatUTC(Date date) {
+ return DATE_FORMATTER.get().format(date);
+ }
+
+ /**
+ * Get the string used for quoting identifiers in this database's SQL dialect.
+ * @param connection the database connection
+ * @return the quote string
+ * @throws SQLException
+ */
+ public static String getIdentifierQuoteString(Connection connection) throws SQLException {
+ String quoteString = connection.getMetaData().getIdentifierQuoteString();
+ quoteString = quoteString == null ? "" : quoteString;
+ return quoteString;
+ }
+
+ /**
+ * Quote the given string.
+ * @param orig the string to quote
+ * @param quote the quote character
+ * @return the quoted string
+ */
+ public static String quoteString(String orig, String quote) {
+ return quote + orig + quote;
+ }
+}
+
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index 073a896..0907d40 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -13,27 +13,55 @@ import java.util.Map;
import java.util.Properties;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
-
import javax.sql.DataSource;
-
import org.apache.rocketmq.connect.jdbc.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import com.alibaba.druid.pool.DruidDataSourceFactory;
-
import org.apache.rocketmq.connect.jdbc.schema.*;
import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
public class Querier {
- static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
+
private final Logger log = LoggerFactory.getLogger(getClass()); // use concrete subclass
protected String topicPrefix;
protected String jdbcUrl;
private final Queue<Connection> connections = new ConcurrentLinkedQueue<>();
- private Config config = new Config();
+ private Config config;
+
+ /**
+ * @return the config
+ */
+ public Config getConfig() {
+ return config;
+ }
+
+ public void setConfig(Config config) {
+ this.config = config;
+ log.info("config load successfully");
+ }
+
private DataSource dataSource;
private List<Table> list = new LinkedList<>();
+ private String mode;
+
+
+ public DataSource getDataSource() {
+ return dataSource;
+ }
+
+ public void setDataSource(DataSource dataSource) {
+ this.dataSource = dataSource;
+ }
+
+ public String getMode() {
+ return mode;
+ }
+
+ public void setMode(String mode) {
+ this.mode = mode;
+ }
+
public List<Table> getList() {
return list;
@@ -44,7 +72,6 @@ public class Querier {
}
public Connection getConnection() throws SQLException {
-
// These config names are the same for both source and sink configs ...
String username = config.jdbcUsername;
String dbPassword = config.jdbcPassword;
@@ -76,7 +103,7 @@ public class Querier {
protected PreparedStatement createDBPreparedStatement(Connection db) throws SQLException {
String SQL = "select table_name,column_name,data_type,column_type,character_set_name "
- + "from information_schema.columns " + "where table_schema = jdbc_db order by ORDINAL_POSITION";
+ + "from information_schema.columns " + "where table_schema = jdbc_db order by ORDINAL_POSITION";
log.trace("Creating a PreparedStatement '{}'", SQL);
PreparedStatement stmt = db.prepareStatement(SQL);
@@ -96,30 +123,29 @@ public class Querier {
return stmt.executeQuery();
}
+ private Schema schema;
+
public static void main(String[] args) throws Exception {
- Querier querier = new Querier();
+ TimestampIncrementingQuerier querier = new TimestampIncrementingQuerier();
try {
querier.start();
querier.poll();
-
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
-
}
- private Schema schema;
-
public void poll() {
try {
PreparedStatement stmt;
String query = "select * from ";
Connection conn = dataSource.getConnection();
-
for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
String db = entry.getKey();
+ if (!db.contains("jdbc_db"))
+ continue;
Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
@@ -155,7 +181,13 @@ public class Querier {
}
public void start() throws Exception {
- initDataSource();
+ try {
+
+ log.info("datasorce success");
+ initDataSource();
+ } catch (Throwable exception) {
+ log.info("error,{}", exception);
+ }
schema = new Schema(dataSource);
schema.load();
log.info("schema load successful");
@@ -163,9 +195,10 @@ public class Querier {
private void initDataSource() throws Exception {
Map<String, String> map = new HashMap<>();
+
map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
map.put("url",
- "jdbc:mysql://" + config.jdbcUrl + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
+ "jdbc:mysql://" + config.jdbcUrl + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
map.put("username", config.jdbcUsername);
map.put("password", config.jdbcPassword);
map.put("initialSize", "2");
@@ -175,9 +208,15 @@ public class Querier {
map.put("minEvictableIdleTimeMillis", "300000");
map.put("validationQuery", "SELECT 1 FROM DUAL");
map.put("testWhileIdle", "true");
- log.info("{},config read successful", map);
- dataSource = DruidDataSourceFactory.createDataSource(map);
-
+ log.info("{} config read successful", map);
+ try {
+ dataSource = DruidDataSourceFactory.createDataSource(map);
+ } catch (Exception exception) {
+ log.info("exeception,{}", exception);
+ } catch (Error e) {
+ log.info("error,{},e", e);
+ }
+ log.info("datasorce success");
}
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
new file mode 100644
index 0000000..1dadc4f
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
@@ -0,0 +1,315 @@
+package org.apache.rocketmq.connect.jdbc.source;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.GregorianCalendar;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.TimeZone;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import javax.sql.DataSource;
+
+import org.apache.rocketmq.connect.jdbc.Config;
+import org.apache.rocketmq.connect.jdbc.schema.Database;
+import org.apache.rocketmq.connect.jdbc.schema.Schema;
+import org.apache.rocketmq.connect.jdbc.schema.Table;
+import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.alibaba.druid.pool.DruidDataSourceFactory;
+
+
+public class TimestampIncrementingQuerier extends Querier {
+ protected PreparedStatement stmt;
+ static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
+ protected String jdbcUrl;
+
+ private Config config;
+ private DataSource dataSource;
+ private static final Logger log = LoggerFactory.getLogger(TimestampIncrementingQuerier.class);
+ private List<Table> list = new LinkedList<>();
+ private HashMap<String, Long> incrementingMap;
+ private HashMap<String, Timestamp> timestampMap;
+ private static final Calendar UTC_CALENDAR = new GregorianCalendar(TimeZone.getTimeZone("UTC+8"));
+ private String timestampColumn = "";
+ static final String INCREMENTING_FIELD = "incrementing";
+ static final String TIMESTAMP_FIELD = "timestamp";
+ private Map<String, Long> offset;
+ private Long timestampOffset;
+ private String incrementingColumn = "";
+ private Map<String, String> partition;
+ private Schema schema;
+
+ public String getTimestampColumn() {
+ return timestampColumn;
+ }
+
+ public void setTimestampColumn(String timestampColumn) {
+ this.timestampColumn = timestampColumn;
+ }
+
+ public String getIncrementingColumn() {
+ return incrementingColumn;
+ }
+
+ public void setIncrementingColumn(String incrementingColumn) {
+ this.incrementingColumn = incrementingColumn;
+ }
+
+ private Long incrementingOffset = null;
+ private String name;
+
+ public void extractRecord(String name) throws SQLException {
+ if (incrementingColumn != null) {
+ log.info("{}", name);
+ incrementingMap.put(name, incrementingOffset);
+ }
+ if (timestampColumn != null) {
+ timestampMap.put(name, new Timestamp(timestampOffset));
+ }
+ }
+
+ @SuppressWarnings("deprecation")
+ public void storeRecord(String name) throws SQLException {
+ offset = new HashMap<>();
+ if (incrementingColumn != null) {
+ Long id = 0L;
+
+ if (incrementingMap.containsKey(name)) {
+ id = incrementingMap.get(name);
+ System.out.println("read incrementingMap" + id);
+ }
+ assert (incrementingOffset == null || id > incrementingOffset) || timestampColumn != null;
+ incrementingOffset = id;
+ offset.put(INCREMENTING_FIELD, id);
+ }
+ if (timestampColumn != null) {
+ Timestamp timestamp = new Timestamp(0);
+ if (timestampMap.containsKey(name))
+ timestamp = timestampMap.get(name);
+
+ System.out.println("read timestampColumn" + timestamp.toString());
+ timestampOffset = timestamp.getTimezoneOffset() + timestamp.getTime();
+ System.out.println("read" + new Timestamp(timestampOffset));
+ offset.put(TIMESTAMP_FIELD, timestampOffset);
+ }
+ log.info("{}store", new Timestamp(timestampOffset));
+ partition = Collections.singletonMap("table", name);
+ }
+
+ protected void createPreparedStatement(Connection conn) throws SQLException {
+ // Default when unspecified uses an autoincrementing column
+ if (incrementingColumn != null && incrementingColumn.isEmpty()) {
+ incrementingColumn = JdbcUtils.getAutoincrementColumn(conn, name);
+ }
+
+ String quoteString = conn.getMetaData().getIdentifierQuoteString();
+ StringBuilder builder = new StringBuilder();
+ builder.append("SELECT * FROM ");
+ builder.append(name);
+
+ quoteString = quoteString == null ? "" : quoteString;
+
+ if (incrementingColumn != null && timestampColumn != null) {
+ // This version combines two possible conditions. The first checks timestamp ==
+ // last
+ // timestamp and incrementing > last incrementing. The timestamp alone would
+ // include
+ // duplicates, but adding the incrementing condition ensures no duplicates, e.g.
+ // you would
+ // get only the row with id = 23:
+ // timestamp 1234, id 22 <- last
+ // timestamp 1234, id 23
+ // The second check only uses the timestamp >= last timestamp. This covers
+ // everything new,
+ // even if it is an update of the existing row. If we previously had:
+ // timestamp 1234, id 22 <- last
+ // and then these rows were written:
+ // timestamp 1235, id 22
+ // timestamp 1236, id 23
+ // We should capture both id = 22 (an update) and id = 23 (a new row)
+ String timeString = quoteString + timestampColumn + quoteString;
+ String incrString = quoteString + incrementingColumn + quoteString;
+ builder.append(" WHERE ");
+ builder.append(timeString);
+ builder.append(" < CURRENT_TIMESTAMP AND ((");
+ builder.append(timeString);
+ builder.append(" = ? AND ");
+ builder.append(incrString);
+ builder.append(" > ?");
+ builder.append(") OR ");
+ builder.append(timeString);
+ builder.append(" > ?)");
+ builder.append(" ORDER BY ");
+ builder.append(timeString);
+ builder.append(",");
+ builder.append(incrString);
+ builder.append(" ASC");
+
+ } else if (incrementingColumn != null) {
+ String incrString = quoteString + incrementingColumn + quoteString;
+ builder.append(" WHERE ");
+ builder.append(incrString);
+ builder.append(" > ?");
+ builder.append(" ORDER BY ");
+ builder.append(incrString);
+ builder.append(" ASC");
+ } else if (timestampColumn != null) {
+ String timeString = quoteString + timestampColumn + quoteString;
+ builder.append(" WHERE ");
+ builder.append(timeString);
+ builder.append(" > ? AND ");
+ builder.append(timeString);
+ builder.append(" < CURRENT_TIMESTAMP ORDER BY ");
+ builder.append(timeString);
+ builder.append(" ASC");
+ }
+ String queryString = builder.toString();
+ stmt = conn.prepareStatement(queryString);
+ log.info(queryString);
+ }
+
+ public static void main(String[] args) throws Exception {
+ TimestampIncrementingQuerier querier = new TimestampIncrementingQuerier();
+ try {
+ querier.start();
+ querier.poll();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ protected ResultSet executeQuery() throws SQLException {
+ if (incrementingColumn != null && timestampColumn != null) {
+ Timestamp ts = new Timestamp(timestampOffset == null ? 0 : timestampOffset);
+ stmt.setTimestamp(1, ts);
+ stmt.setLong(2, (incrementingOffset == null ? -1 : incrementingOffset));
+ stmt.setTimestamp(3, ts);
+ } else if (incrementingColumn != null) {
+ stmt.setLong(1, (incrementingOffset == null ? -1 : incrementingOffset));
+ } else if (timestampColumn != null) {
+ Timestamp ts = new Timestamp(timestampOffset == null ? 0 : timestampOffset);
+ stmt.setTimestamp(1, ts);
+ }
+ log.info("{}·", stmt);
+ log.info("{},{}", incrementingOffset, timestampOffset);
+ return stmt.executeQuery();
+ }
+
+ public List<Table> getList() {
+ return list;
+ }
+
+ public void setList(List<Table> list) {
+ this.list = list;
+ }
+
+ public void poll() {
+ try {
+ list.clear();
+ Connection conn = dataSource.getConnection();
+ for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
+ String db = entry.getKey();
+ if (!db.contains("time_db"))
+ continue;
+ log.info("{} database is loading", db);
+ Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<String, Table> tableEntry = iterator.next();
+ String tb = tableEntry.getKey();
+ log.info("{} table is loading", tb);
+ name = db + "." + tb;
+ storeRecord(name);
+ createPreparedStatement(conn);
+ ResultSet rs;
+ rs = executeQuery();
+ List<String> colList = tableEntry.getValue().getColList();
+ List<String> DataTypeList = tableEntry.getValue().getRawDataTypeList();
+ List<ColumnParser> ParserList = tableEntry.getValue().getParserList();
+
+ while (rs.next()) {
+ Table table = new Table(db, tb);
+ System.out.print("|");
+ table.setColList(colList);
+ table.setRawDataTypeList(DataTypeList);
+ table.setParserList(ParserList);
+ for (String string : colList) {
+ table.getDataList().add(rs.getObject(string));
+ System.out.print(string + " : " + rs.getObject(string) + "|");
+ }
+ incrementingOffset = incrementingOffset > rs.getInt(incrementingColumn) ? incrementingOffset
+ : rs.getInt(incrementingColumn);
+ timestampOffset = timestampOffset > rs.getTimestamp(timestampColumn).getTime() ? timestampOffset
+ : rs.getTimestamp(timestampColumn).getTime();
+ System.out.println(timestampOffset);
+ list.add(table);
+ System.out.println();
+ }
+ extractRecord(name);
+ incrementingOffset = 0L;
+ timestampOffset = 0L;
+ }
+ }
+ conn.close();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void start() throws Exception {
+ try {
+ initDataSource();
+ if (incrementingColumn != null && timestampColumn != null) {
+ incrementingMap = new HashMap<>();
+ timestampMap = new HashMap<>();
+ } else if (incrementingColumn != null) {
+ incrementingMap = new HashMap<>();
+ } else if (timestampColumn != null) {
+ timestampMap = new HashMap<>();
+ }
+ } catch (Throwable exception) {
+ log.info("error,{}", exception);
+ }
+ schema = new Schema(dataSource);
+ schema.load();
+ }
+
+ private void initDataSource() throws Exception {
+ Map<String, String> map = new HashMap<>();
+ config = super.getConfig();
+ timestampColumn = config.getTimestampColmnName();
+ incrementingColumn = config.getIncrementingColumnName();
+ map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
+ map.put("url",
+ "jdbc:mysql://" + config.jdbcUrl + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
+ map.put("username", config.jdbcUsername);
+ map.put("password", config.jdbcPassword);
+ map.put("initialSize", "2");
+ map.put("maxActive", "2");
+ map.put("maxWait", "60000");
+ map.put("timeBetweenEvictionRunsMillis", "60000");
+ map.put("minEvictableIdleTimeMillis", "300000");
+ map.put("validationQuery", "SELECT 1 FROM DUAL");
+ map.put("testWhileIdle", "true");
+ log.info("{}config read successfully", map);
+ try {
+ dataSource = DruidDataSourceFactory.createDataSource(map);
+ } catch (Exception exception) {
+ log.info("exeception,{}", exception);
+ } catch (Error error) {
+ log.info("error,{}", error);
+ }
+ log.info("datasorce success");
+ }
+}
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java
index f9c8c6f..429494b 100644
--- a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java
@@ -16,29 +16,88 @@
*/
package org.apache.rocketmq.connect.jdbc.connector;
+
import java.util.Collection;
-import org.junit.Test;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.sql.DataSource;
+import org.junit.Test;
+import java.sql.*;
+import com.alibaba.druid.pool.DruidDataSourceFactory;
import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.data.SourceDataEntry;
import io.openmessaging.internal.DefaultKeyValue;
public class JdbcSourceTaskTest {
+ KeyValue kv;
+ DataSource dataSource;
+
+ @Test
+ public void testBulk() throws InterruptedException {
+ KeyValue kv = new DefaultKeyValue();
+ kv.put("jdbcUrl", "localhost:3306");
+ kv.put("jdbcUsername", "root");
+ kv.put("jdbcPassword", "199812160");
+ kv.put("mode", "bulk");
+ kv.put("rocketmqTopic", "JdbcTopic");
+ JdbcSourceTask task = new JdbcSourceTask();
+ task.start(kv);
+ Collection<SourceDataEntry> sourceDataEntry = task.poll();
+ System.out.println(sourceDataEntry);
+ }
+
+ @Test
+ public void testTimestampIncrementing() throws InterruptedException, SQLException {
+ kv = new DefaultKeyValue();
+ kv.put("jdbcUrl", "localhost:3306");
+ kv.put("jdbcUsername", "root");
+ kv.put("jdbcPassword", "199812160");
+ kv.put("incrementingColumnName", "id");
+ kv.put("timestampColmnName", "timestamp");
+ kv.put("mode", "incrementing+timestamp");
+ kv.put("rocketmqTopic", "JdbcTopic");
+ JdbcSourceTask task = new JdbcSourceTask();
+ task.start(kv);
+ Collection<SourceDataEntry> sourceDataEntry = task.poll();
+ System.out.println(sourceDataEntry);
+ Map<String, String> map = new HashMap<>();
+ map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
+ map.put("url", "jdbc:mysql://" + kv.getString("jdbcUrl")
+ + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
+ map.put("username", kv.getString("jdbcUsername"));
+ map.put("password", kv.getString("jdbcPassword"));
+ map.put("initialSize", "2");
+ map.put("maxActive", "2");
+ map.put("maxWait", "60000");
+ map.put("timeBetweenEvictionRunsMillis", "60000");
+ map.put("minEvictableIdleTimeMillis", "300000");
+ map.put("validationQuery", "SELECT 1 FROM DUAL");
+ map.put("testWhileIdle", "true");
+ try {
+ dataSource = DruidDataSourceFactory.createDataSource(map);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ Connection connection= dataSource.getConnection();
+ PreparedStatement statement;
+ String s="insert into time_db.timestamp_tb (name) values(\"test\")";
+ statement=connection.prepareStatement(s);
+ statement.executeUpdate();
- @Test
- public void test() throws InterruptedException {
- KeyValue kv = new DefaultKeyValue();
- kv.put("jdbcUrl","localhost:3306");
- kv.put("jdbcUsername","root");
- kv.put("jdbcPassword","199812160");
- kv.put("mode","bulk");
- kv.put("rocketmqTopic","JdbcTopic");
- JdbcSourceTask task = new JdbcSourceTask();
- task.start(kv);
- Collection<SourceDataEntry> sourceDataEntry = task.poll();
- System.out.println(sourceDataEntry);
-
- }
+ sourceDataEntry = task.poll();
+ System.out.println(sourceDataEntry);
+ s="update time_db.timestamp_tb set name=\"liu\" where id < 2";
+ statement=connection.prepareStatement(s);
+ statement.executeUpdate();
+ sourceDataEntry = task.poll();
+ System.out.println(sourceDataEntry);
+ task.stop();
+
+ connection.close();
+ }
}
[rocketmq-connect] 42/43: Merge branch 'master' of github.com:apache/rocketmq-externals
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 6708ada617d9f6cfef5ca42a3c2f97af44603a89
Merge: 7ebfedf 5f40a88
Author: xxxxx <xx...@qq.com>
AuthorDate: Fri Jun 4 12:13:13 2021 +0800
Merge branch 'master' of github.com:apache/rocketmq-externals
* 'master' of github.com:apache/rocketmq-externals: (69 commits)
[#715] Support the RocketMQ TableSource based on the new Source interface (#716)
#720 fix rocketmq console fail to update nameserver address (#721)
[ISSUE #711] Consumers can skip the accumulation of messages. (#712)
[#705] Support the implementation of new Source interface (#706)
fix(ssl): SelfSignedCertificate work well with JDK15
fix(jdk15): resolve aspecj exception
[Replicator] Restart of connector causes message duplication problem & Unit test bug fix (#694)
[Replicator] Fix message duplication problem (#692)
Support query message by page (#688)
[ISSUE #682] Fix wrong message trace detail (#683)
Support DLQ topic resend message (#653)
Fix too much warn log because non-exist retry topic (#681)
Update README.md (#553)
Update producer.html (#675)
Fix dashboard notification when set rocketmq.config.enableDashBoardCollect=false (#679)
[rocketmq-connector-flink] rebalance cause offset rollback to long time ago (#672)
[ISSUE #666]Upgrade RocketMQ version (#667)
Update .travis.yml
Update .travis.yml
Optimize dependencies and support JDK 11+
...
README.md | 81 ++++++++++++++--------
pom.xml | 3 +-
.../rocketmq/connect/jdbc/common/CloneUtils.java | 28 ++++++++
.../rocketmq/connect/jdbc/common/ConstDefine.java | 2 +-
.../rocketmq/connect/jdbc/common/DBUtils.java | 31 +++++----
.../rocketmq/connect/jdbc/config/Config.java | 13 ++--
.../connect/jdbc/config/SinkDbConnectorConfig.java | 11 +--
.../jdbc/config/SourceDbConnectorConfig.java | 4 +-
.../connect/jdbc/connector/JdbcSinkConnector.java | 63 +++++++++--------
.../jdbc/connector/JdbcSourceConnector.java | 2 +-
.../connect/jdbc/connector/JdbcSourceTask.java | 9 ++-
.../apache/rocketmq/connect/jdbc/sink/Updater.java | 22 +++---
.../rocketmq/connect/jdbc/source/Querier.java | 7 +-
.../jdbc/source/TimestampIncrementingQuerier.java | 6 +-
.../connect/jdbc/strategy/DivideTaskByQueue.java | 3 +-
.../connect/jdbc/strategy/DivideTaskByTopic.java | 25 ++++---
16 files changed, 195 insertions(+), 115 deletions(-)
[rocketmq-connect] 05/43: Update JdbcSourceConnector.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 675099dc7f85460317aa60031039132d4fabec09
Author: Yuchen Li <yu...@126.com>
AuthorDate: Fri Jul 19 14:22:14 2019 +0800
Update JdbcSourceConnector.java
---
.../apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
index d6b0e3b..8a6047c 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
@@ -36,16 +36,13 @@ public class JdbcSourceConnector extends SourceConnector {
@Override
public String verifyAndSetConfig(KeyValue config) {
- System.out.println(11+config.toString());
log.info("JdbcSourceConnector verifyAndSetConfig enter");
for (String requestKey : Config.REQUEST_CONFIG) {
- System.out.println(12+requestKey);
if (!config.containsKey(requestKey)) {
return "Request config key: " + requestKey;
}
}
this.config = config;
- System.out.println(11+config.toString());
return "";
}
@@ -78,6 +75,5 @@ public class JdbcSourceConnector extends SourceConnector {
config.add(this.config);
return config;
}
-
- // abstract Set<String> getRequiredConfig();
+
}
[rocketmq-connect] 36/43: [ISSUE #545]bug fix (#546)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 0b56e472982adb92ba7fa52bf2d3c9b5f326e969
Author: mike_xwm <mi...@126.com>
AuthorDate: Wed Apr 1 13:54:08 2020 +0800
[ISSUE #545]bug fix (#546)
Co-authored-by: MajorHe1 <he...@gmail.com>
---
pom.xml | 2 +
.../rocketmq/connect/jdbc/common/CloneUtils.java | 28 +++++++++++
.../rocketmq/connect/jdbc/common/ConstDefine.java | 2 +-
.../rocketmq/connect/jdbc/common/DBUtils.java | 2 +-
.../rocketmq/connect/jdbc/config/Config.java | 31 ------------
.../connect/jdbc/config/SinkDbConnectorConfig.java | 11 +++--
.../jdbc/config/SourceDbConnectorConfig.java | 4 +-
.../connect/jdbc/connector/JdbcSinkConnector.java | 56 ++++++++++++----------
.../connect/jdbc/connector/JdbcSourceTask.java | 9 ++--
.../apache/rocketmq/connect/jdbc/sink/Updater.java | 22 +++++----
.../rocketmq/connect/jdbc/source/Querier.java | 7 ++-
.../jdbc/source/TimestampIncrementingQuerier.java | 6 +--
.../connect/jdbc/strategy/DivideTaskByQueue.java | 3 +-
.../connect/jdbc/strategy/DivideTaskByTopic.java | 14 ++++--
14 files changed, 107 insertions(+), 90 deletions(-)
diff --git a/pom.xml b/pom.xml
index 9df23c4..61680f1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -190,6 +190,7 @@
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-connector</artifactId>
<version>0.1.1</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>io.openmessaging</groupId>
@@ -264,6 +265,7 @@
<artifactId>druid</artifactId>
<version>1.0.31</version>
</dependency>
+
</dependencies>
</project>
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/common/CloneUtils.java b/src/main/java/org/apache/rocketmq/connect/jdbc/common/CloneUtils.java
new file mode 100644
index 0000000..f0ff98e
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/common/CloneUtils.java
@@ -0,0 +1,28 @@
+package org.apache.rocketmq.connect.jdbc.common;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+public class CloneUtils {
+ @SuppressWarnings("unchecked")
+ public static <T extends Serializable> T clone(T obj) {
+ T clonedObj = null;
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos);
+ oos.writeObject(obj);
+ oos.close();
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ ObjectInputStream ois = new ObjectInputStream(bais);
+ clonedObj = (T) ois.readObject();
+ ois.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ return clonedObj;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java b/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java
index f49d367..e6d2f7a 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java
@@ -19,5 +19,5 @@ package org.apache.rocketmq.connect.jdbc.common;
public class ConstDefine {
public static String JDBC_CONNECTOR_ADMIN_PREFIX = "JDBC-CONNECTOR-ADMIN";
-
+ public static final String PREFIX = "jdbc";
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java b/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
index ab58153..31a86d1 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
@@ -196,7 +196,7 @@ public class DBUtils {
map.put("username", config.getDbUsername());
map.put("password", config.getDbPassword());
map.put("initialSize", "1");
- map.put("maxActive", "1");
+ map.put("maxActive", "2");
map.put("maxWait", "60000");
map.put("timeBetweenEvictionRunsMillis", "60000");
map.put("minEvictableIdleTimeMillis", "300000");
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index cca1aa5..9162bad 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
@@ -78,9 +78,6 @@ public class Config {
private long timestampDelayInterval = 0;
private String dbTimezone = "GMT+8";
private String queueName;
- private String jdbcUrl;
- private String jdbcUsername;
- private String jdbcPassword;
private Logger log = LoggerFactory.getLogger(Config.class);
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
@@ -327,32 +324,4 @@ public class Config {
public void setWhiteTable(String whiteTable) {
this.whiteTable = whiteTable;
}
-
- public void setPollInterval(long pollInterval) {
- this.pollInterval = pollInterval;
- }
-
- public String getJdbcUrl() {
- return jdbcUrl;
- }
-
- public void setJdbcUrl(String jdbcUrl) {
- this.jdbcUrl = jdbcUrl;
- }
-
- public String getJdbcUsername() {
- return jdbcUsername;
- }
-
- public void setJdbcUsername(String jdbcUsername) {
- this.jdbcUsername = jdbcUsername;
- }
-
- public String getJdbcPassword() {
- return jdbcPassword;
- }
-
- public void setJdbcPassword(String jdbcPassword) {
- this.jdbcPassword = jdbcPassword;
- }
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
index 3ff4f71..26b1541 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
@@ -16,16 +16,16 @@ public class SinkDbConnectorConfig extends DbConnectorConfig{
private String srcNamesrvs;
private String srcCluster;
private long refreshInterval;
- private Map<String, List<TaskTopicInfo>> topicRouteMap;
+ private Map<String, Set<TaskTopicInfo>> topicRouteMap;
public SinkDbConnectorConfig(){
}
@Override
public void validate(KeyValue config) {
- this.taskParallelism = config.getInt(Config.CONN_TASK_PARALLELISM, 0);
+ this.taskParallelism = config.getInt(Config.CONN_TASK_PARALLELISM, 1);
- int strategy = config.getInt(Config.CONN_TASK_DIVIDE_STRATEGY, DivideStrategyEnum.BY_QUEUE.ordinal());
+ int strategy = config.getInt(Config.CONN_TASK_DIVIDE_STRATEGY, DivideStrategyEnum.BY_TOPIC.ordinal());
if (strategy == DivideStrategyEnum.BY_QUEUE.ordinal()) {
this.taskDivideStrategy = new DivideTaskByQueue();
} else {
@@ -43,6 +43,7 @@ public class SinkDbConnectorConfig extends DbConnectorConfig{
this.srcNamesrvs = config.getString(Config.CONN_SOURCE_RMQ);
this.srcCluster = config.getString(Config.CONN_SOURCE_CLUSTER);
this.refreshInterval = config.getLong(Config.REFRESH_INTERVAL, 3);
+ this.mode = config.getString(Config.CONN_DB_MODE, "bulk");
}
@@ -81,11 +82,11 @@ public class SinkDbConnectorConfig extends DbConnectorConfig{
return this.refreshInterval;
}
- public Map<String, List<TaskTopicInfo>> getTopicRouteMap() {
+ public Map<String, Set<TaskTopicInfo>> getTopicRouteMap() {
return topicRouteMap;
}
- public void setTopicRouteMap(Map<String, List<TaskTopicInfo>> topicRouteMap) {
+ public void setTopicRouteMap(Map<String, Set<TaskTopicInfo>> topicRouteMap) {
this.topicRouteMap = topicRouteMap;
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java
index 801e411..4972739 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java
@@ -18,9 +18,9 @@ public class SourceDbConnectorConfig extends DbConnectorConfig{
@Override
public void validate(KeyValue config) {
- this.taskParallelism = config.getInt(Config.CONN_TASK_PARALLELISM, 0);
+ this.taskParallelism = config.getInt(Config.CONN_TASK_PARALLELISM, 1);
- int strategy = config.getInt(Config.CONN_TASK_DIVIDE_STRATEGY, DivideStrategyEnum.BY_QUEUE.ordinal());
+ int strategy = config.getInt(Config.CONN_TASK_DIVIDE_STRATEGY, DivideStrategyEnum.BY_TOPIC.ordinal());
if (strategy == DivideStrategyEnum.BY_QUEUE.ordinal()) {
this.taskDivideStrategy = new DivideTaskByQueue();
} else {
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
index 0f818ee..6a41646 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
@@ -3,46 +3,48 @@ package org.apache.rocketmq.connect.jdbc.connector;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.sink.SinkConnector;
-import org.apache.commons.lang3.StringUtils;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
-import org.apache.commons.lang3.text.StrSubstitutor;
import org.apache.rocketmq.client.exception.MQClientException;
-import org.apache.rocketmq.common.MixAll;
-import org.apache.rocketmq.common.TopicConfig;
-import org.apache.rocketmq.common.protocol.body.TopicList;
import org.apache.rocketmq.common.protocol.route.BrokerData;
import org.apache.rocketmq.common.protocol.route.QueueData;
import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.connect.jdbc.common.CloneUtils;
import org.apache.rocketmq.connect.jdbc.common.ConstDefine;
import org.apache.rocketmq.connect.jdbc.common.Utils;
-import org.apache.rocketmq.connect.jdbc.config.*;
+import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.config.DataType;
+import org.apache.rocketmq.connect.jdbc.config.DbConnectorConfig;
+import org.apache.rocketmq.connect.jdbc.config.SinkDbConnectorConfig;
+import org.apache.rocketmq.connect.jdbc.config.TaskDivideConfig;
+import org.apache.rocketmq.connect.jdbc.config.TaskTopicInfo;
import org.apache.rocketmq.remoting.RPCHook;
-import org.apache.rocketmq.remoting.exception.RemotingException;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
public class JdbcSinkConnector extends SinkConnector {
private static final Logger log = LoggerFactory.getLogger(JdbcSinkConnector.class);
private DbConnectorConfig dbConnectorConfig;
private volatile boolean configValid = false;
private ScheduledExecutorService executor;
- private Map<String, List<TaskTopicInfo>> topicRouteMap;
+ private HashMap<String, Set<TaskTopicInfo>> topicRouteMap;
private DefaultMQAdminExt srcMQAdminExt;
private volatile boolean adminStarted;
public JdbcSinkConnector() {
- topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
+ topicRouteMap = new HashMap<>();
dbConnectorConfig = new SinkDbConnectorConfig();
executor = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("JdbcSinkConnector-SinkWatcher-%d").daemon(true).build());
}
@@ -93,30 +95,34 @@ public class JdbcSinkConnector extends SinkConnector {
public void startListener() {
executor.scheduleAtFixedRate(new Runnable() {
+ boolean first = true;
+ HashMap<String, Set<TaskTopicInfo>> origin = null;
+
@Override
public void run() {
- Map<String, List<TaskTopicInfo>> origin = topicRouteMap;
- topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
-
buildRoute();
-
+ if (first) {
+ origin = CloneUtils.clone(topicRouteMap);
+ first = false;
+ }
if (!compare(origin, topicRouteMap)) {
context.requestTaskReconfiguration();
+ origin = CloneUtils.clone(topicRouteMap);
}
}
}, ((SinkDbConnectorConfig) dbConnectorConfig).getRefreshInterval(), ((SinkDbConnectorConfig) dbConnectorConfig).getRefreshInterval(), TimeUnit.SECONDS);
}
- public boolean compare(Map<String, List<TaskTopicInfo>> origin, Map<String, List<TaskTopicInfo>> updated) {
+ public boolean compare(Map<String, Set<TaskTopicInfo>> origin, Map<String, Set<TaskTopicInfo>> updated) {
if (origin.size() != updated.size()) {
return false;
}
- for (Map.Entry<String, List<TaskTopicInfo>> entry : origin.entrySet()) {
+ for (Map.Entry<String, Set<TaskTopicInfo>> entry : origin.entrySet()) {
if (!updated.containsKey(entry.getKey())) {
return false;
}
- List<TaskTopicInfo> originTasks = entry.getValue();
- List<TaskTopicInfo> updateTasks = updated.get(entry.getKey());
+ Set<TaskTopicInfo> originTasks = entry.getValue();
+ Set<TaskTopicInfo> updateTasks = updated.get(entry.getKey());
if (originTasks.size() != updateTasks.size()) {
return false;
}
@@ -145,7 +151,7 @@ public class JdbcSinkConnector extends SinkConnector {
TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
if (!topicRouteMap.containsKey(topic)) {
- topicRouteMap.put(topic, new ArrayList<TaskTopicInfo>());
+ topicRouteMap.put(topic, new HashSet<>(16));
}
for (QueueData qd : topicRouteData.getQueueDatas()) {
if (brokerNameSet.contains(qd.getBrokerName())) {
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index d533395..f36623f 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -21,12 +21,15 @@ package org.apache.rocketmq.connect.jdbc.connector;
import io.openmessaging.connector.api.source.SourceTask;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.rocketmq.connect.jdbc.common.ConstDefine;
import org.apache.rocketmq.connect.jdbc.config.Config;
import org.apache.rocketmq.connect.jdbc.common.DBUtils;
import org.apache.rocketmq.connect.jdbc.config.ConfigUtil;
@@ -105,13 +108,13 @@ public class JdbcSourceTask extends SourceTask {
.entryType(EntryType.UPDATE);
for (int i = 0; i < dataRow.getColList().size(); i++) {
Object[] value = new Object[2];
- value[0] = value[1] = dataRow.getDataList().get(i);
+ value[0] = value[1] = dataRow.getParserList().get(i).getValue(dataRow.getDataList().get(i));
dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSONObject.toJSONString(value));
}
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap((config.getDbUrl() + config.getDbPort()).getBytes("UTF-8")),
- ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8")));
+ ByteBuffer.wrap((ConstDefine.PREFIX + config.getDbUrl() + config.getDbPort()).getBytes(StandardCharsets.UTF_8)),
+ ByteBuffer.wrap(jsonObject.toJSONString().getBytes(StandardCharsets.UTF_8)));
res.add(sourceDataEntry);
log.debug("sourceDataEntry : {}", JSONObject.toJSONString(sourceDataEntry));
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java b/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
index e30c65f..9feffe6 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
@@ -49,7 +49,7 @@ public class Updater {
isSuccess = true;
// 再查原有数据是否存在,存在则删除
beforeUpdateId = queryBeforeUpdateRowId(dbName, tableName, fieldMap);
- if (beforeUpdateId != 0){
+ if (beforeUpdateId != 0 && afterUpdateId != beforeUpdateId){
isSuccess = deleteRow(dbName, tableName, beforeUpdateId);
}
break;
@@ -107,7 +107,7 @@ public class Updater {
ResultSet rs;
PreparedStatement stmt;
Boolean finishQuery = false;
- String query = "select id from " + dbName + "." + tableName + " where ";
+ String query = "select id from " + dbName + "." + tableName + " where 1=1";
for (Map.Entry<Field, Object[]> entry : fieldMap.entrySet()) {
count ++;
@@ -116,7 +116,7 @@ public class Updater {
Object fieldValue = entry.getValue()[0];
if ("id".equals(fieldName))
continue;
- if (count != 1) {
+ if (count <=fieldMap.size()) {
query += " and ";
}
if (fieldValue == null)
@@ -150,7 +150,7 @@ public class Updater {
ResultSet rs;
PreparedStatement stmt;
Boolean finishQuery = false;
- String query = "select id from " + dbName + "." + tableName + " where ";
+ String query = "select id from " + dbName + "." + tableName + " where 1=1";
for (Map.Entry<Field, Object[]> entry : fieldMap.entrySet()) {
count ++;
@@ -159,7 +159,7 @@ public class Updater {
Object fieldValue = entry.getValue()[1];
if ("id".equals(fieldName))
continue;
- if (count != 1) {
+ if (count <=fieldMap.size()) {
query += " and ";
}
if (fieldValue == null)
@@ -200,19 +200,21 @@ public class Updater {
FieldType fieldType = entry.getKey().getType();
Object fieldValue = entry.getValue()[1];
if ("id".equals(fieldName)) {
- if (id == 0)
+ if (id == 0){
+ if(count==fieldMap.size()) update = update.substring(0,update.length()-1);
continue;
- else
+ }else{
fieldValue = id;
- }
- if (count != 1) {
- update += ", ";
+ }
}
if (fieldValue == null) {
update += fieldName + " = NULL";
} else {
update = typeParser(fieldType, fieldName, fieldValue, update);
}
+ if(count<fieldMap.size()){
+ update += ",";
+ }
}
try {
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index d2544f9..03447a8 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -92,12 +92,12 @@ public class Querier {
public void poll() {
try {
PreparedStatement stmt;
- StringBuilder query = new StringBuilder("select * from ");
LinkedList<Table> tableLinkedList = new LinkedList<>();
for (Map.Entry<String, Database> entry : schema.getDbMap().entrySet()) {
String db = entry.getKey();
Iterator<Map.Entry<String, Table>> iterator = entry.getValue().getTableMap().entrySet().iterator();
while (iterator.hasNext()) {
+ StringBuilder query = new StringBuilder("select * from ");
Map.Entry<String, Table> tableEntry = iterator.next();
String tb = tableEntry.getKey();
query.append(db + "." + tb);
@@ -116,7 +116,7 @@ public class Querier {
query.append(condition);
}
}
- stmt = connection.prepareStatement(query + db + "." + tb);
+ stmt = connection.prepareStatement(query.toString());
ResultSet rs;
rs = stmt.executeQuery();
List<String> colList = tableEntry.getValue().getColList();
@@ -158,7 +158,7 @@ public class Querier {
for (String whiteTableName : whiteTableObject.keySet()){
Collections.addAll(whiteTableSet, whiteTableName);
HashMap<String, String> filterMap = new HashMap<>();
- JSONObject tableFilterObject = (JSONObject)whiteTableObject.get(whiteTableName);
+ JSONObject tableFilterObject = JSONObject.parseObject(whiteTableObject.get(whiteTableName).toString());
for(String filterKey : tableFilterObject.keySet()){
filterMap.put(filterKey, tableFilterObject.getString(filterKey));
}
@@ -170,5 +170,4 @@ public class Querier {
schema.load();
log.info("load schema success");
}
-
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
index 964322d..0ab72df 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
@@ -288,9 +288,9 @@ public class TimestampIncrementingQuerier extends Querier {
incrementingColumn = config.getIncrementingColumnName();
map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
map.put("url",
- "jdbc:mysql://" + config.getJdbcUrl() + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
- map.put("username", config.getJdbcUsername());
- map.put("password", config.getJdbcPassword());
+ "jdbc:mysql://" + config.getDbUrl() + ":" + config.getDbPort() +"?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
+ map.put("username", config.getDbUsername());
+ map.put("password", config.getDbPassword());
map.put("initialSize", "2");
map.put("maxActive", "2");
map.put("maxWait", "60000");
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
index 797710a..9d23fd2 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
@@ -19,6 +19,7 @@ package org.apache.rocketmq.connect.jdbc.strategy;
import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.internal.DefaultKeyValue;
+import java.util.Set;
import org.apache.rocketmq.connect.jdbc.config.*;
import java.util.ArrayList;
@@ -41,7 +42,7 @@ public class DivideTaskByQueue extends TaskDivideStrategy {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
- Map<String, List<TaskTopicInfo>> topicRouteMap = ((SinkDbConnectorConfig)dbConnectorConfig).getTopicRouteMap();
+ Map<String, Set<TaskTopicInfo>> topicRouteMap = ((SinkDbConnectorConfig)dbConnectorConfig).getTopicRouteMap();
int id = -1;
for (String t : topicRouteMap.keySet()) {
for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
index 762c7a0..c1d5020 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
@@ -52,7 +52,11 @@ public class DivideTaskByTopic extends TaskDivideStrategy {
String filter = entry.getValue();
Map<String, String> tableMap = new HashMap<>();
tableMap.put(tableKey, filter);
- taskTopicList.get(ind).put(dbKey, tableMap);
+ if(!taskTopicList.get(ind).containsKey(dbKey)){
+ taskTopicList.get(ind).put(dbKey, tableMap);
+ }else {
+ taskTopicList.get(ind).get(dbKey).putAll(tableMap);
+ }
}
for (int i = 0; i < parallelism; i++) {
@@ -77,11 +81,13 @@ public class DivideTaskByTopic extends TaskDivideStrategy {
int parallelism = tdc.getTaskParallelism();
int id = -1;
Set<String> topicRouteSet = ((SinkDbConnectorConfig)dbConnectorConfig).getWhiteTopics();
- Map<Integer, String> taskTopicList = new HashMap<>();
+ Map<Integer, StringBuilder> taskTopicList = new HashMap<>();
for (String topicName : topicRouteSet) {
int ind = ++id % parallelism;
if (!taskTopicList.containsKey(ind)) {
- taskTopicList.put(ind, topicName);
+ taskTopicList.put(ind, new StringBuilder(topicName));
+ }else {
+ taskTopicList.get(ind).append(",").append(topicName);
}
}
@@ -91,7 +97,7 @@ public class DivideTaskByTopic extends TaskDivideStrategy {
keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
- keyValue.put(Config.CONN_TOPIC_NAMES, JSONObject.toJSONString(taskTopicList.get(i)));
+ keyValue.put(Config.CONN_TOPIC_NAMES, taskTopicList.get(i).toString());
keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
[rocketmq-connect] 30/43: [ISSUE #495] jdbc-sink-connector support divide task by queue (#496)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 252e26ae3dfc5b78600c9a1e2f67a2e98917e55b
Author: surilli <94...@qq.com>
AuthorDate: Tue Dec 31 15:40:35 2019 +0800
[ISSUE #495] jdbc-sink-connector support divide task by queue (#496)
* rocketmq-connect-jdbc add JdbcSinkTask & optimize JdbcSourceTask
* jdbc-connector bug-fix duplicate data pushed in table
* [ISSUE #489] JDBC Connector support divide task by topic strategy
* [ISSUE #489] JDBC Connector support divide task by topic strategy
* [ISSUE #495] jdbc-sink-connector support divide task by queue
Co-authored-by: Xiongmengfei <Xi...@163.com>
---
pom.xml | 42 +++++--
.../rocketmq/connect/jdbc/common/ConstDefine.java | 23 ++++
.../rocketmq/connect/jdbc/common/DBUtils.java | 6 +-
.../apache/rocketmq/connect/jdbc/common/Utils.java | 74 +++++++++++
.../rocketmq/connect/jdbc/config/Config.java | 4 +
.../connect/jdbc/config/SinkDbConnectorConfig.java | 32 ++++-
.../connect/jdbc/connector/JdbcSinkConnector.java | 135 ++++++++++++++++++++-
.../connect/jdbc/strategy/DivideTaskByQueue.java | 29 +++--
8 files changed, 318 insertions(+), 27 deletions(-)
diff --git a/pom.xml b/pom.xml
index 1d708f3..59442c5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -16,7 +16,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-connect-jdbc</artifactId>
- <version>1.0.0</version>
+ <version>0.0.1-SNAPSHOT</version>
<name>connect-jdbc</name>
<url>https://github.com/apache/incubator-rocketmq-externals/tree/master/rocketmq-connect-jdbc</url>
@@ -40,6 +40,7 @@
<!-- Compiler settings properties -->
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
+ <rocketmq.version>4.5.2</rocketmq.version>
</properties>
<build>
@@ -188,7 +189,7 @@
<dependency>
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-connector</artifactId>
- <version>0.1.0-beta</version>
+ <version>0.1.1-beta-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>io.openmessaging</groupId>
@@ -198,7 +199,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
- <version>1.2.51</version>
+ <version>1.2.60</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
@@ -217,14 +218,25 @@
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-client</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-tools</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-remoting</artifactId>
+ <version>${rocketmq.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-openmessaging</artifactId>
<version>4.3.2</version>
</dependency>
- <dependency>
- <groupId>com.alibaba</groupId>
- <artifactId>druid</artifactId>
- <version>1.0.18</version>
- </dependency>
+
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
@@ -239,7 +251,19 @@
<groupId>io.javalin</groupId>
<artifactId>javalin</artifactId>
<version>1.3.0</version>
- </dependency>
+ </dependency>
+
+ <dependency>
+ <groupId>com.github.shyiko</groupId>
+ <artifactId>mysql-binlog-connector-java</artifactId>
+ <version>0.20.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>druid</artifactId>
+ <version>1.0.31</version>
+ </dependency>
</dependencies>
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java b/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java
new file mode 100644
index 0000000..f49d367
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.common;
+
+public class ConstDefine {
+
+ public static String JDBC_CONNECTOR_ADMIN_PREFIX = "JDBC-CONNECTOR-ADMIN";
+
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java b/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
index ccee96b..ab58153 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
@@ -192,9 +192,9 @@ public class DBUtils {
Map<String, String> map = new HashMap<>();
map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
map.put("url",
- "jdbc:mysql://" + config.getJdbcUrl() + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8&characterEncoding=utf8");
- map.put("username", config.getJdbcUsername());
- map.put("password", config.getJdbcPassword());
+ "jdbc:mysql://" + config.getDbUrl() + ":" + config.getDbPort() + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8&characterEncoding=utf8");
+ map.put("username", config.getDbUsername());
+ map.put("password", config.getDbPassword());
map.put("initialSize", "1");
map.put("maxActive", "1");
map.put("maxWait", "60000");
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/common/Utils.java b/src/main/java/org/apache/rocketmq/connect/jdbc/common/Utils.java
new file mode 100644
index 0000000..5708e34
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/common/Utils.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.common;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class Utils {
+ private static final Logger log = LoggerFactory.getLogger(Utils.class);
+
+ public static String createGroupName(String prefix) {
+ return new StringBuilder().append(prefix).append("-").append(System.currentTimeMillis()).toString();
+ }
+
+ public static String createGroupName(String prefix, String postfix) {
+ return new StringBuilder().append(prefix).append("-").append(postfix).toString();
+ }
+
+ public static String createTaskId(String prefix) {
+ return new StringBuilder().append(prefix).append("-").append(System.currentTimeMillis()).toString();
+ }
+
+ public static String createInstanceName(String namesrvAddr) {
+ String[] namesrvArray = namesrvAddr.split(";");
+ List<String> namesrvList = new ArrayList<>();
+ for (String ns : namesrvArray) {
+ if (!namesrvList.contains(ns)) {
+ namesrvList.add(ns);
+ }
+ }
+ Collections.sort(namesrvList);
+ return String.valueOf(namesrvList.toString().hashCode());
+ }
+
+ public static List<BrokerData> examineBrokerData(DefaultMQAdminExt defaultMQAdminExt, String topic,
+ String cluster) throws RemotingException, MQClientException, InterruptedException {
+ List<BrokerData> brokerList = new ArrayList<>();
+
+ TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic);
+ if (topicRouteData.getBrokerDatas() != null) {
+ for (BrokerData broker : topicRouteData.getBrokerDatas()) {
+ if (StringUtils.equals(broker.getCluster(), cluster)) {
+ brokerList.add(broker);
+ }
+ }
+ }
+ return brokerList;
+ }
+
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index 91a3e51..ae86d3f 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
@@ -59,6 +59,10 @@ public class Config {
public static final String CONN_TOPIC_NAMES = "topicNames";
public static final String CONN_DB_MODE = "mode";
+ public static final String CONN_SOURCE_RMQ = "source-rocketmq";
+ public static final String CONN_SOURCE_CLUSTER = "source-cluster";
+ public static final String REFRESH_INTERVAL = "refresh.interval";
+
/* Mode Config */
private String mode = "";
private String incrementingColumnName = "";
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
index 851b253..3ff4f71 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
@@ -6,11 +6,17 @@ import org.apache.rocketmq.connect.jdbc.strategy.DivideTaskByQueue;
import org.apache.rocketmq.connect.jdbc.strategy.DivideTaskByTopic;
import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Set;
public class SinkDbConnectorConfig extends DbConnectorConfig{
private Set<String> whiteList;
+ private String srcNamesrvs;
+ private String srcCluster;
+ private long refreshInterval;
+ private Map<String, List<TaskTopicInfo>> topicRouteMap;
public SinkDbConnectorConfig(){
}
@@ -34,11 +40,15 @@ public class SinkDbConnectorConfig extends DbConnectorConfig{
this.dbUserName = config.getString(Config.CONN_DB_USERNAME);
this.dbPassword = config.getString(Config.CONN_DB_PASSWORD);
+ this.srcNamesrvs = config.getString(Config.CONN_SOURCE_RMQ);
+ this.srcCluster = config.getString(Config.CONN_SOURCE_CLUSTER);
+ this.refreshInterval = config.getLong(Config.REFRESH_INTERVAL, 3);
+
}
private void buildWhiteList(KeyValue config) {
this.whiteList = new HashSet<>();
- String whiteListStr = config.getString(Config.CONN_WHITE_LIST, "");
+ String whiteListStr = config.getString(Config.CONN_TOPIC_NAMES, "");
String[] wl = whiteListStr.trim().split(",");
if (wl.length <= 0)
throw new IllegalArgumentException("White list must be not empty.");
@@ -59,6 +69,26 @@ public class SinkDbConnectorConfig extends DbConnectorConfig{
this.whiteList = whiteList;
}
+ public String getSrcNamesrvs() {
+ return this.srcNamesrvs;
+ }
+
+ public String getSrcCluster() {
+ return this.srcCluster;
+ }
+
+ public long getRefreshInterval() {
+ return this.refreshInterval;
+ }
+
+ public Map<String, List<TaskTopicInfo>> getTopicRouteMap() {
+ return topicRouteMap;
+ }
+
+ public void setTopicRouteMap(Map<String, List<TaskTopicInfo>> topicRouteMap) {
+ this.topicRouteMap = topicRouteMap;
+ }
+
@Override
public Set<String> getWhiteTopics() {
return getWhiteList();
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
index 935ad52..0f818ee 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
@@ -3,22 +3,69 @@ package org.apache.rocketmq.connect.jdbc.connector;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.sink.SinkConnector;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.commons.lang3.text.StrSubstitutor;
+import org.apache.rocketmq.client.exception.MQClientException;
+import org.apache.rocketmq.common.MixAll;
+import org.apache.rocketmq.common.TopicConfig;
+import org.apache.rocketmq.common.protocol.body.TopicList;
+import org.apache.rocketmq.common.protocol.route.BrokerData;
+import org.apache.rocketmq.common.protocol.route.QueueData;
+import org.apache.rocketmq.common.protocol.route.TopicRouteData;
+import org.apache.rocketmq.connect.jdbc.common.ConstDefine;
+import org.apache.rocketmq.connect.jdbc.common.Utils;
import org.apache.rocketmq.connect.jdbc.config.*;
+import org.apache.rocketmq.remoting.RPCHook;
+import org.apache.rocketmq.remoting.exception.RemotingException;
+import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
public class JdbcSinkConnector extends SinkConnector {
private static final Logger log = LoggerFactory.getLogger(JdbcSinkConnector.class);
- private KeyValue config;
private DbConnectorConfig dbConnectorConfig;
private volatile boolean configValid = false;
+ private ScheduledExecutorService executor;
+ private Map<String, List<TaskTopicInfo>> topicRouteMap;
- public JdbcSinkConnector(){
+ private DefaultMQAdminExt srcMQAdminExt;
+
+ private volatile boolean adminStarted;
+
+ public JdbcSinkConnector() {
+ topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
dbConnectorConfig = new SinkDbConnectorConfig();
+ executor = Executors.newSingleThreadScheduledExecutor(new BasicThreadFactory.Builder().namingPattern("JdbcSinkConnector-SinkWatcher-%d").daemon(true).build());
+ }
+
+ private synchronized void startMQAdminTools() {
+ if (!configValid || adminStarted) {
+ return;
+ }
+ RPCHook rpcHook = null;
+ this.srcMQAdminExt = new DefaultMQAdminExt(rpcHook);
+ this.srcMQAdminExt.setNamesrvAddr(((SinkDbConnectorConfig) this.dbConnectorConfig).getSrcNamesrvs());
+ this.srcMQAdminExt.setAdminExtGroup(Utils.createGroupName(ConstDefine.JDBC_CONNECTOR_ADMIN_PREFIX));
+ this.srcMQAdminExt.setInstanceName(Utils.createInstanceName(((SinkDbConnectorConfig) this.dbConnectorConfig).getSrcNamesrvs()));
+
+ try {
+ this.srcMQAdminExt.start();
+ log.info("RocketMQ srcMQAdminExt started");
+
+ } catch (MQClientException e) {
+ log.error("Replicator start failed for `srcMQAdminExt` exception.", e);
+ }
+
+ adminStarted = true;
}
@Override
@@ -40,7 +87,80 @@ public class JdbcSinkConnector extends SinkConnector {
@Override
public void start() {
+ startMQAdminTools();
+ startListener();
+ }
+
+ public void startListener() {
+ executor.scheduleAtFixedRate(new Runnable() {
+ @Override
+ public void run() {
+ Map<String, List<TaskTopicInfo>> origin = topicRouteMap;
+ topicRouteMap = new HashMap<String, List<TaskTopicInfo>>();
+
+ buildRoute();
+
+ if (!compare(origin, topicRouteMap)) {
+ context.requestTaskReconfiguration();
+ }
+ }
+ }, ((SinkDbConnectorConfig) dbConnectorConfig).getRefreshInterval(), ((SinkDbConnectorConfig) dbConnectorConfig).getRefreshInterval(), TimeUnit.SECONDS);
+ }
+ public boolean compare(Map<String, List<TaskTopicInfo>> origin, Map<String, List<TaskTopicInfo>> updated) {
+ if (origin.size() != updated.size()) {
+ return false;
+ }
+ for (Map.Entry<String, List<TaskTopicInfo>> entry : origin.entrySet()) {
+ if (!updated.containsKey(entry.getKey())) {
+ return false;
+ }
+ List<TaskTopicInfo> originTasks = entry.getValue();
+ List<TaskTopicInfo> updateTasks = updated.get(entry.getKey());
+ if (originTasks.size() != updateTasks.size()) {
+ return false;
+ }
+
+ if (!originTasks.containsAll(updateTasks)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ public void buildRoute() {
+ String srcCluster = ((SinkDbConnectorConfig) this.dbConnectorConfig).getSrcCluster();
+ try {
+ for (String topic : ((SinkDbConnectorConfig) this.dbConnectorConfig).getWhiteList()) {
+
+ // different from BrokerData with cluster field, which can ensure the brokerData is from expected cluster.
+ // QueueData use brokerName as unique info on cluster of rocketmq. so when we want to get QueueData of
+ // expected cluster, we should get brokerNames of expected cluster, and then filter queueDatas.
+ List<BrokerData> brokerList = Utils.examineBrokerData(this.srcMQAdminExt, topic, srcCluster);
+ Set<String> brokerNameSet = new HashSet<String>();
+ for (BrokerData b : brokerList) {
+ brokerNameSet.add(b.getBrokerName());
+ }
+
+ TopicRouteData topicRouteData = srcMQAdminExt.examineTopicRouteInfo(topic);
+ if (!topicRouteMap.containsKey(topic)) {
+ topicRouteMap.put(topic, new ArrayList<TaskTopicInfo>());
+ }
+ for (QueueData qd : topicRouteData.getQueueDatas()) {
+ if (brokerNameSet.contains(qd.getBrokerName())) {
+ for (int i = 0; i < qd.getReadQueueNums(); i++) {
+ TaskTopicInfo taskTopicInfo = new TaskTopicInfo(topic, qd.getBrokerName(), i, null);
+ topicRouteMap.get(topic).add(taskTopicInfo);
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Fetch topic list error.", e);
+ } finally {
+ srcMQAdminExt.shutdown();
+ }
}
@Override
@@ -70,6 +190,10 @@ public class JdbcSinkConnector extends SinkConnector {
return new ArrayList<KeyValue>();
}
+ startMQAdminTools();
+
+ buildRoute();
+
TaskDivideConfig tdc = new TaskDivideConfig(
this.dbConnectorConfig.getDbUrl(),
this.dbConnectorConfig.getDbPort(),
@@ -80,6 +204,9 @@ public class JdbcSinkConnector extends SinkConnector {
this.dbConnectorConfig.getTaskParallelism(),
this.dbConnectorConfig.getMode()
);
+
+ ((SinkDbConnectorConfig) this.dbConnectorConfig).setTopicRouteMap(topicRouteMap);
+
return this.dbConnectorConfig.getTaskDivideStrategy().divide(this.dbConnectorConfig, tdc);
}
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
index 7ef5c31..797710a 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
@@ -19,10 +19,7 @@ package org.apache.rocketmq.connect.jdbc.strategy;
import com.alibaba.fastjson.JSONObject;
import io.openmessaging.KeyValue;
import io.openmessaging.internal.DefaultKeyValue;
-import org.apache.rocketmq.replicator.config.DataType;
-import org.apache.rocketmq.replicator.config.TaskConfigEnum;
-import org.apache.rocketmq.replicator.config.TaskDivideConfig;
-import org.apache.rocketmq.replicator.config.TaskTopicInfo;
+import org.apache.rocketmq.connect.jdbc.config.*;
import java.util.ArrayList;
import java.util.HashMap;
@@ -30,12 +27,21 @@ import java.util.List;
import java.util.Map;
public class DivideTaskByQueue extends TaskDivideStrategy {
+
@Override
- public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
+ public List<KeyValue> divide(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
+ if (dbConnectorConfig instanceof SinkDbConnectorConfig){
+ return divideSinkTaskByQueue(dbConnectorConfig, tdc);
+ }
+ return null;
+ }
+
+ public List<KeyValue> divideSinkTaskByQueue(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
List<KeyValue> config = new ArrayList<KeyValue>();
int parallelism = tdc.getTaskParallelism();
Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
+ Map<String, List<TaskTopicInfo>> topicRouteMap = ((SinkDbConnectorConfig)dbConnectorConfig).getTopicRouteMap();
int id = -1;
for (String t : topicRouteMap.keySet()) {
for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
@@ -49,11 +55,14 @@ public class DivideTaskByQueue extends TaskDivideStrategy {
for (int i = 0; i < parallelism; i++) {
KeyValue keyValue = new DefaultKeyValue();
- keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
- keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
- keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
- keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
- keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
+ keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
+ keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
+ keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
+ keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
+ keyValue.put(Config.CONN_TOPIC_NAMES, JSONObject.toJSONString(queueTopicList.get(i)));
+ keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
+ keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
+ keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
config.add(keyValue);
}
[rocketmq-connect] 28/43: [ISSUE #498] update rocketmq-jdbc-connector README.md (#499)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 759e422296ec0cc193786f88449c214750f4017a
Author: MajorHe1 <53...@users.noreply.github.com>
AuthorDate: Thu Dec 26 20:55:27 2019 +0800
[ISSUE #498] update rocketmq-jdbc-connector README.md (#499)
---
README.md | 255 ++++++++++----------------------------------------------------
1 file changed, 40 insertions(+), 215 deletions(-)
diff --git a/README.md b/README.md
index 1970378..7b3d397 100644
--- a/README.md
+++ b/README.md
@@ -1,235 +1,60 @@
-# RocketMQ-connect-jdbc
+# rocketmq-connect-jdbc
-### Directory Structure Description
-
-```web-idl
-│ pom.xml
-│ README.md
-└─src
- ├─main
- │ └─java
- │ └─org
- │ └─apache
- │ └─rocketmq
- │ └─connect
- │ └─jdbc
- │ │ Config.java
- │ ├─connector
- │ │ JdbcSourceConnector.java
- │ │ JdbcSourceTask.java
- │ ├─dialect
- │ ├─schema
- │ │ │ Database.java
- │ │ │ Schema.java
- │ │ │ Table.java
- │ │ │
- │ │ └─column
- │ │ BigIntColumnParser.java
- │ │ ColumnParser.java
- │ │ DateTimeColumnParser.java
- │ │ DefaultColumnParser.java
- │ │ EnumColumnParser.java
- │ │ IntColumnParser.java
- │ │ SetColumnParser.java
- │ │ StringColumnParser.java
- │ │ TimeColumnParser.java
- │ │ YearColumnParser.java
- │ ├─sink
- │ └─source
- │ Querier.java
- └─test
- └─java
- └─org
- └─apache
- └─rocketmq
- └─connect
- └─jdbc
- └─connector
- JdbcSourceConnectorTest.java
- JdbcSourceTaskTest.java
-```
-
-### Some Result of Testing JdbcSourceTask
-
-
-
-#### Data Type：SourceDataEntry
-
-{sourcePartition,sourcePosition,DataEntry{timestamp,entryType=CREATE,queueName,shardingKey,schema.schema=Schema{dataSource=DATABASE_NAME,name=TABLE_NAME,fields=[Field{index,name,type}]},payloading}}
-
-- For example
-
-```javascript
- SourceDataEntry{sourcePartition=java.nio.HeapByteBuffer[pos=0 lim=14 cap=14], sourcePosition=java.nio.HeapByteBuffer[pos=0 lim=44 cap=44]} DataEntry{timestamp=1564397062419, entryType=CREATE, queueName='student', shardingKey='null',
- schema=Schema{dataSource='jdbc_db', name='student', fields=[Field{index=0, name='id', type=INT32}, Field{index=1, name='first', type=STRING},
- Field{index=2, name='last', type=STRING}, Field{index=3, name='age', type=INT32}]}, payload=[102121, "Python", "Py", 25]}
+## rocketmq-connect-jdbc 打包
```
-
-#### Mentioned DataBase Information and all SourceDataEntry
-
-- For example
-
-
-
-
-
-**启动Connector**
-
-[http://127.0.0.1:8081/connectors/connector-name?config={"connector-class":"org.apache.rocketmq.connect.kafka.connector.KafkaSourceConnector","oms-driver-url":"oms](http://127.0.0.1:8081/connectors/connector-name?config=%7B%22connector-class%22:%22org.apache.rocketmq.connect.kafka.connector.KafkaSourceConnector%22,%22oms-driver-url%22:%22oms): rocketmq://127.0.0.1:9876/default:default","tasks.num":"1","kafka.topics":"test1,test2","kafka.group.id":"group0","kafka.bootstrap.server":"127.0. [...]
-
-**查看Connector运行状态**
-
-<http://127.0.0.1:8081/connectors/connector-name/status>
-
-**查看Connector配置**
-
-<http://127.0.0.1:8081/connectors/connector-name/config>
-
-**关闭Connector**
-
-<http://127.0.0.1:8081/connectors/connector-name/stop>
-
-
-
-
-
-
-
-# JDBC Connector 构建
-
-
-
-#### 一、下载rocketmq-connect-runtime
-
+mvn clean install -DskipTest -U
```
-1、git clone https://github.com/apache/rocketmq-externals.git
-
-2、cd rocketmq-externals/rocketmq-connect-runtime
-3、mvn -Dmaven.test.skip=true package
-
-4、cd target/distribution/conf
-```
+## rocketmq-connect-jdbc 启动
-- a、修改connect.conf配置文件
+* **jdbc-source-connector** 启动
```
-#1、rocketmq 配置
-namesrvAddr=127.0.0.1:9876
-
-#2、file-connect jar包路径
-pluginPaths=/home/connect/file-connect/target
-
-#3、runtime持久化文件目录
-storePathRootDir=/home/connect/storeRoot
-
-#4、http服务端口
-httpPort=8081
+http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-source-connector-name}
+?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector",“dbUrl”:"${source-db-ip}",dbPort”:"${source-db-port}",dbUsername”:"${source-db-username}",dbPassword”:"${source-db-password}","rocketmqTopic":"jdbcTopic","mode":"bulk","whiteDataBase":{"${source-db-name}":{"${source-table-name}":{"${source-column-name}":"${source-column-value}"}}},"source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
```
-b、日志相关配置在logback.xml中修改
+* **jdbc-sink-connector** 启动
```
-注：rocketmq需要先创建cluster-topic，config-topic，offset-topic，position-topic
-4个topic，并且为了保证消息有序，每个topic可以只一个queue
+http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-sink-connector-name}
+?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSinkConnector",“dbUrl”:"${sink-db-ip}",dbPort”:"${sink-db-port}",dbUsername”:"${sink-db-username}",dbPassword”:"${sink-db-password}","rocketmqTopic":"jdbcTopic","mode":"bulk","topicNames":"${sink-topic-name}","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
```
+>**注：** `rocketmq-jdbc-connect` 的启动依赖于`rocketmq-connect-runtime`项目的启动，需将打好的`jar`包放置到`runtime`项目中`pluginPaths`配置的路径后再执行上面的启动请求,该值配置在`runtime`项目下的`connect.conf`文件中
-### 二、启动Connector
-
-1、启动runtime
-回到rocketmq-externals/rocketmq-connect-runtime目录
+## rocketmq-connect-jdbc 停止
```
-./run_worker.sh
+http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-connector-name}/stop
```
-看到日志目录查看connect_runtime.log
-
-windows用户可以用CMD到程序根目录下再输入：
-
-```
-cd target/distribution/
-
-java -cp .;./conf/;./lib/* org.apache.rocketmq.connect.runtime.ConnectStartup -c conf/connect.conf
-```
-
-如果看到以下日志说明runttiime启动成功了
-
-2019-07-16 10:56:24 INFO RebalanceService - RebalanceService service started
-2019-07-16 10:56:24 INFO main - The worker [DEFAULT_WORKER_1] boot success.
-
-2、启动sourceConnector
-
-```
-1、git clone https://github.com/apache/rocketmq-externals.git
-
-2、cd rocketmq-externals/rocketmq-connect-jdbc
-
-3、mvn -Dmaven.test.skip=true package
-
-```
-
-- 复制第三方jar至target
-
-```
-mvn dependency:copy-dependencies
-```
-
-#### Bulk查询方法，在http中输入Get 请求（目前仅适配过MYSQL）
-
-请将jdbcUrl, jdbcUsername, jdbcPassword，改为所连接数据库的配置
-
-```http
-http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"bulk","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
-```
-
-看到一下日志说明Jdbc source connector启动成功了
-
-2019-08-09 11:33:22 INFO RebalanceService - JdbcSourceConnector verifyAndSetConfig enter
-2019-08-09 11:33:23 INFO pool-9-thread-1 - Config.load.start
-2019-08-09 11:33:23 INFO pool-9-thread-1 - querier.start
-2019-08-09 11:33:23 INFO pool-9-thread-1 - {password=199812160, validationQuery=SELECT 1 FROM DUAL, testWhileIdle=true, timeBetweenEvictionRunsMillis=60000, minEvictableIdleTimeMillis=300000, initialSize=2, driverClassName=com.mysql.cj.jdbc.Driver, maxWait=60000, url=jdbc:mysql://localhost:3306?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8, username=root, maxActive=2},config read successful
-2019-08-09 11:33:24 INFO RebalanceService - JdbcSourceConnector verifyAndSetConfig enter
-2019-08-09 11:33:25 INFO pool-9-thread-1 - {dataSource-1} inited
-2019-08-09 11:33:27 INFO pool-9-thread-1 - schema load successful
-2019-08-09 11:33:27 INFO pool-9-thread-1 - querier.poll
-
-#### Incrementing and / or Timestamp Querier
-
-- 要使用的时间戳和/或自增列必须在连接器处理的所有表上。如果不同的表具有不同名称的时间戳/自增列，则需要创建单独的连接器配置；
-- 可以结合使用这些方法中的（时间戳/自增）或两者（时间戳+自增）；
-
-##### Incrementing Querier （自增列查询）
-
-注：表中需要含递增的一列，如果只使用自增列，则不会捕获对数据的更新，除非每次更新时自增列也会增加。
-
-```http
-http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"incrementing","incrementingColumnName":"id,"source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
-```
-
-##### Timestamp Querier （时间戳查询）
-
-
-```http
-http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"timestamp":"timestampColumnName","id","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
-```
-
-##### Incrementing + Timestamp Querier (自增列+时间戳查询)
-
-
-```http
-http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"incrementing+timestamp","timestampColumnName":"timestamp","incrementingColumnName":"id","source-record-converter":"org.apache.rocketmq.connect.runtime.converter [...]
-```
-
-日志如果显示为如下，则connect成功。
-
-2019-08-15 14:09:43 INFO RebalanceService - JdbcSourceConnector verifyAndSetConfig enter
-2019-08-15 14:09:44 INFO pool-14-thread-1 - Config.load.start
-2019-08-15 14:09:44 INFO pool-14-thread-1 - config load successfully
-2019-08-15 14:09:44 INFO pool-14-thread-1 - map start,199812160
-2019-08-15 14:09:44 INFO pool-14-thread-1 - {password=199812160, validationQuery=SELECT 1 FROM DUAL, testWhileIdle=true, timeBetweenEvictionRunsMillis=60000, minEvictableIdleTimeMillis=300000, initialSize=2, driverClassName=com.mysql.cj.jdbc.Driver, maxWait=60000, url=jdbc:mysql://127.0.0.1:3306?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8, username=root, maxActive=2}config read successfully
+## rocketmq-connect-jdbc 参数说明
+* **jdbc-source-connector 参数说明**
-3、启动sinkConnector
+参数 | 类型 | 是否必须 | 描述 | 样例
+|---|---|---|---|---|
+|dbUrl | String | 是 | source端 DB ip | 192.168.1.2|
+|dbPort | String | 是 | source端 DB port | 3306 |
+|dbUsername | String | 是 | source端 DB 用户名 | root |
+|dbPassword | String | 是 | source端 DB 密码 | 123456 |
+|whiteDataBase | String | 是 | source端同步数据白名单，嵌套配置，为{DB名：{表名：{字段名：字段值}}}，若无指定字段数据同步，字段名可设为NO-FILTER，值为任意 | {"DATABASE_TEST":{"TEST_DATA":{"name":"test"}}} |
+|mode | String | 是 | source-connector 模式，目前仅支持bulk | bulk |
+|~~rocketmqTopic~~ | String | 是 | 待废弃 | jdbcTopic |
+|task-divide-strategy | Integer | 否 | task 分配策略, 默认值为 0，表示按照topic分配任务，每一个table便是一个topic | 0 |
+|task-parallelism | Integer | 否 | task parallelism，默认值为 1，表示将topic拆分为多少个任务进行执行 | 2 |
+|source-record-converter | String | 是 | source data 解析 | org.apache.rocketmq.connect.runtime.converter.JsonConverter |
-To Be Continued.
+* **jdbc-sink-connector 参数说明**
+参数 | 类型 | 是否必须 | 描述 | 样例
+|---|---|---|---|---|
+|dbUrl | String | 是 | sink端 DB ip | 192.168.1.2|
+|dbPort | String | 是 | sink端 DB port | 3306 |
+|dbUsername | String | 是 | sink端 DB 用户名 | root |
+|dbPassword | String | 是 | sink端 DB 密码 | 123456 |
+|topicNames | String | 是 | sink端同步数据的topic名字 | topic-1,topic-2 |
+|mode | String | 是 | source-connector 模式，目前仅支持bulk | bulk |
+|~~rocketmqTopic~~ | String | 是 | 待废弃 | jdbcTopic |
+|task-divide-strategy | Integer | 否 | task 分配策略, 默认值为 0，表示按照topic分配任务，每一个table便是一个topic | 0 |
+|task-parallelism | Integer | 否 | task parallelism，默认值为 1，表示将topic拆分为多少个任务进行执行 | 2 |
+|source-record-converter | String | 是 | source data 解析 | org.apache.rocketmq.connect.runtime.converter.JsonConverter |
[rocketmq-connect] 06/43: Add Jdbc Source Task (To be continued)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 3c5f659d7093c9f6d43019b99047bca2dbc3869a
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Fri Jul 19 14:34:09 2019 +0800
Add Jdbc Source Task (To be continued)
---
.../connect/jdbc/connector/JdbcSourceTask.java | 28 ++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
new file mode 100644
index 0000000..adb1c62
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -0,0 +1,28 @@
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.connector;
+
+import io.openmessaging.connector.api.source.SourceTask;
+
+public abstract class JdbcSourceTask extends SourceTask {
+/*
+ * To Be Continued
+ */
+}
\ No newline at end of file
[rocketmq-connect] 08/43: Add JdbcSourceTask
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 5a83890b55770820325d32aa7c86df6b477c37b7
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Mon Jul 29 21:20:48 2019 +0800
Add JdbcSourceTask
---
.../org/apache/rocketmq/connect/jdbc/Config.java | 12 +-
.../apache/rocketmq/connect/jdbc/Replicator.java | 118 +++++++++++++
.../connect/jdbc/connector/JdbcSourceTask.java | 107 +++++++++++-
.../rocketmq/connect/jdbc/schema/Database.java | 100 +++++++++++
.../rocketmq/connect/jdbc/schema/Schema.java | 128 ++++++++++++++
.../apache/rocketmq/connect/jdbc/schema/Table.java | 90 ++++++++++
.../column/BigIntColumnParser.java} | 40 ++++-
.../connect/jdbc/schema/column/ColumnParser.java | 104 ++++++++++++
.../jdbc/schema/column/DateTimeColumnParser.java | 53 ++++++
.../column/DefaultColumnParser.java} | 27 ++-
.../column/EnumColumnParser.java} | 36 +++-
.../jdbc/schema/column/IntColumnParser.java | 66 ++++++++
.../jdbc/schema/column/SetColumnParser.java | 54 ++++++
.../jdbc/schema/column/StringColumnParser.java | 57 +++++++
.../column/TimeColumnParser.java} | 29 +++-
.../column/YearColumnParser.java} | 30 +++-
.../rocketmq/connect/jdbc/source/Querier.java | 187 +++++++++++++++++++++
17 files changed, 1181 insertions(+), 57 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
index 217b449..69ff9b0 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
@@ -31,9 +31,9 @@ public class Config {
private static final Logger LOG = LoggerFactory.getLogger(Config.class);
/* Database Connection Config */
- public String jdbcUrl;
- public String jdbcUsername;
- public String jdbcPassword;
+ public String jdbcUrl="localhost:3306";
+ public String jdbcUsername="root";
+ public String jdbcPassword="199812160";
public String rocketmqTopic;
public String jdbcBackoff;
public String jdbcAttempts;
@@ -54,7 +54,7 @@ public class Config {
/*Connector config*/
public String tableTypes="table";
- public int pollInterval=5000;
+ public long pollInterval=5000;
public int batchMaxRows=100;
public long tablePollInterval=60000;
public long timestampDelayInterval=0;
@@ -67,7 +67,7 @@ public class Config {
add("jdbcUsername");
add("jdbcPassword");
add("mode");
- add("queueName");
+ add("rocketmqTopic");
}
};
@@ -278,7 +278,7 @@ public class Config {
this.tableTypes = tableTypes;
}
- public int getPollInterval() {
+ public long getPollInterval() {
return pollInterval;
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Replicator.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Replicator.java
new file mode 100644
index 0000000..b24b7e5
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/Replicator.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Replicator {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(Replicator.class);
+
+ private static final Logger POSITION_LOGGER = LoggerFactory.getLogger("PositionLogger");
+
+ private Config config;
+
+ private EventProcessor eventProcessor;
+
+ private Object lock = new Object();
+ private BinlogPosition nextBinlogPosition;
+ private long nextQueueOffset;
+ private long xid;
+ private BlockingQueue<Transaction> queue = new LinkedBlockingQueue<>();
+
+ public Replicator(Config config){
+ this.config = config;
+ }
+
+ public void start() {
+
+ try {
+
+ eventProcessor = new EventProcessor(this);
+ eventProcessor.start();
+
+ } catch (Exception e) {
+ LOGGER.error("Start error.", e);
+ }
+ }
+
+ public void stop(){
+ eventProcessor.stop();
+ }
+
+ public void commit(Transaction transaction, boolean isComplete) {
+
+ queue.add(transaction);
+ for (int i = 0; i < 3; i++) {
+ try {
+ if (isComplete) {
+ long offset = 1;
+ synchronized (lock) {
+ xid = transaction.getXid();
+ nextBinlogPosition = transaction.getNextBinlogPosition();
+ nextQueueOffset = offset;
+ }
+
+ } else {
+ }
+ break;
+
+ } catch (Exception e) {
+ LOGGER.error("Push error,retry:" + (i + 1) + ",", e);
+ }
+ }
+ }
+
+ public void logPosition() {
+
+ String binlogFilename = null;
+ long xid = 0L;
+ long nextPosition = 0L;
+ long nextOffset = 0L;
+
+ synchronized (lock) {
+ if (nextBinlogPosition != null) {
+ xid = this.xid;
+ binlogFilename = nextBinlogPosition.getBinlogFilename();
+ nextPosition = nextBinlogPosition.getPosition();
+ nextOffset = nextQueueOffset;
+ }
+ }
+
+ if (binlogFilename != null) {
+ POSITION_LOGGER.info("XID: {}, BINLOG_FILE: {}, NEXT_POSITION: {}, NEXT_OFFSET: {}",
+ xid, binlogFilename, nextPosition, nextOffset);
+ }
+
+ }
+
+ public Config getConfig() {
+ return config;
+ }
+
+// public BinlogPosition getNextBinlogPosition() {
+// return nextBinlogPosition;
+// }
+
+ public BlockingQueue<Transaction> getQueue() {
+ return queue;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index adb1c62..32ea763 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -18,11 +18,106 @@
*/
package org.apache.rocketmq.connect.jdbc.connector;
-
import io.openmessaging.connector.api.source.SourceTask;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.rocketmq.connect.jdbc.Config;
+import org.apache.rocketmq.connect.jdbc.Replicator;
+import org.apache.rocketmq.connect.jdbc.schema.Table;
+import org.apache.rocketmq.connect.jdbc.source.Querier;
+import org.apache.rocketmq.connect.jdbc.schema.column.*;
-public abstract class JdbcSourceTask extends SourceTask {
-/*
- * To Be Continued
- */
-}
\ No newline at end of file
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.fastjson.JSON;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.data.EntryType;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.SourceDataEntry;
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.connector.api.data.DataEntryBuilder;
+import io.openmessaging.connector.api.data.Field;
+
+public class JdbcSourceTask extends SourceTask {
+
+ private static final Logger log = LoggerFactory.getLogger(JdbcSourceTask.class);
+
+ private Replicator replicator;
+
+ private Config config;
+
+ private List<Table> list=new LinkedList<>();
+
+ Querier querier = new Querier();
+ @Override
+ public Collection<SourceDataEntry> poll() {
+ List<SourceDataEntry> res = new ArrayList<>();
+ try {
+ JSONObject jsonObject = new JSONObject();
+ jsonObject.put("nextQuery", "database");
+ jsonObject.put("nextPosition", "10");
+ //To be Continued
+ querier.poll();
+ System.out.println(querier.getList().size());
+ for(Table dataRow : querier.getList()){
+ System.out.println(dataRow.getColList().get(0));
+ Schema schema = new Schema();
+ schema.setDataSource(dataRow.getDatabase());
+ schema.setName(dataRow.getName());
+ schema.setFields(new ArrayList<>());
+ for(int i = 0; i < dataRow.getColList().size(); i++){
+ String columnName = dataRow.getColList().get(i);
+ String rawDataType = dataRow.getRawDataTypeList().get(i);
+ Field field = new Field(i, columnName, ColumnParser.mapConnectorFieldType(rawDataType));
+ schema.getFields().add(field);
+ }
+ DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
+ dataEntryBuilder.timestamp(System.currentTimeMillis())
+ .queue(dataRow.getName())
+ .entryType(EntryType.CREATE);
+ for(int i = 0; i < dataRow.getColList().size(); i++){
+ Object value=dataRow.getDataList().get(i);
+ System.out.println(1);
+ System.out.println(dataRow.getColList().get(i)+"|"+value);
+ dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSON.toJSONString(value));
+ }
+ SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
+ ByteBuffer.wrap(config.jdbcUrl.getBytes("UTF-8")),
+ ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8")));
+ res.add(sourceDataEntry);
+ }
+ } catch (Exception e) {
+ log.error("JDBC task poll error, current config:" + JSON.toJSONString(config), e);
+ }
+ return res;
+ }
+
+ @Override
+ public void start(KeyValue props) {
+ try {
+ this.config = new Config();
+ this.config.load(props);
+ querier.start();
+ } catch (Exception e) {
+ log.error("JDBC task start failed.", e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ replicator.stop();
+ }
+
+ @Override public void pause() {
+
+ }
+
+ @Override public void resume() {
+
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
new file mode 100644
index 0000000..15fb77b
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.schema;
+
+//import io.openmessaging.mysql.binlog.EventProcessor;
+import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.sql.DataSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Database {
+ private static final String SQL = "select table_name,column_name,data_type,column_type,character_set_name " +
+ "from information_schema.columns " +
+ "where table_schema = ? order by ORDINAL_POSITION";
+ private String name;
+ private DataSource dataSource;
+ public Map<String, Table> tableMap = new HashMap<String, Table>();
+ public Database(String name, DataSource dataSource) {
+ this.name = name;
+ this.dataSource = dataSource;
+ }
+
+ public void init() throws SQLException {
+ Connection conn = null;
+ PreparedStatement ps = null;
+ ResultSet rs = null;
+
+ try {
+ conn = dataSource.getConnection();
+
+ ps = conn.prepareStatement(SQL);
+ ps.setString(1, name);
+ rs = ps.executeQuery();
+
+ while (rs.next()) {
+ String tableName = rs.getString(1);
+ String colName = rs.getString(2);
+ String dataType = rs.getString(3);
+ String colType = rs.getString(4);
+ String charset = rs.getString(5);
+
+ ColumnParser columnParser = ColumnParser.getColumnParser(dataType, colType, charset);
+
+ if (!tableMap.containsKey(tableName)) {
+ addTable(tableName);
+ }
+ Table table = tableMap.get(tableName);
+ table.addCol(colName);
+ table.addParser(columnParser);
+ table.addRawDataType(dataType);
+ }
+
+ } finally {
+ if (conn != null) {
+ conn.close();
+ }
+ if (ps != null) {
+ ps.close();
+ }
+ if (rs != null) {
+ rs.close();
+ }
+ }
+
+ }
+
+ private void addTable(String tableName) {
+
+ // LOGGER.info("Schema load -- DATABASE:{},\tTABLE:{}", name, tableName);
+
+ Table table = new Table(name, tableName);
+ tableMap.put(tableName, table);
+ }
+
+ public Table getTable(String tableName) {
+
+ return tableMap.get(tableName);
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
new file mode 100644
index 0000000..6ce6621
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.schema;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import javax.sql.DataSource;
+//import io.openmessaging.mysql.binlog.EventProcessor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Schema {
+// private static final Logger LOGGER = LoggerFactory.getLogger(EventProcessor.class);
+
+ private static final String SQL = "select schema_name from information_schema.schemata";
+ //取得数据库
+ private static final List<String> IGNORED_DATABASES = new ArrayList<>(
+ Arrays.asList(new String[] {"information_schema", "mysql", "performance_schema", "sys"})
+ );
+ //忽略的数据库
+ private DataSource dataSource;
+
+ public Map<String, Database> dbMap;
+
+ public Schema(DataSource dataSource) {
+ this.dataSource = dataSource;
+ }
+
+ public void load() throws SQLException {
+
+ dbMap = new HashMap<>();
+
+ Connection conn = null;
+ PreparedStatement ps = null;
+ ResultSet rs = null;
+
+ try {
+ conn = dataSource.getConnection();
+
+ ps = conn.prepareStatement(SQL);
+ rs = ps.executeQuery();
+
+ while (rs.next()) {
+ String dbName = rs.getString(1);
+ if (!IGNORED_DATABASES.contains(dbName)) {
+ Database database = new Database(dbName, dataSource);
+ dbMap.put(dbName, database);
+ //dbMap存着各个数据库的名字
+ }
+ }
+
+ } finally {
+
+ if (conn != null) {
+ conn.close();
+ }
+ if (ps != null) {
+ ps.close();
+ }
+ if (rs != null) {
+ rs.close();
+ }
+ }
+
+ for (Database db : dbMap.values()) {
+ db.init();
+ }
+
+ }
+
+ public Table getTable(String dbName, String tableName) {
+
+ if (dbMap == null) {
+ reload();
+ }
+
+ Database database = dbMap.get(dbName);
+ if (database == null) {
+ return null;
+ }
+
+ Table table = database.getTable(tableName);
+ if (table == null) {
+ return null;
+ }
+
+ return table;
+ }
+
+ private void reload() {
+
+ while (true) {
+ try {
+ load();
+ break;
+ } catch (Exception e) {
+ // LOGGER.error("Reload schema error.", e);
+ System.out.println("Reload schema error."+e);
+ }
+ }
+ }
+
+ public void reset() {
+ dbMap = null;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
new file mode 100644
index 0000000..c0d793d
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.schema;
+
+import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
+import java.util.LinkedList;
+import java.util.List;
+
+public class Table {
+
+ private String database;
+ private String name;
+ private List<String> colList = new LinkedList<>();
+ private List<ColumnParser> parserList = new LinkedList<>();
+ private List<String> rawDataTypeList = new LinkedList<>();
+ private List<Object> dataList =new LinkedList<>();
+
+ public Table(String database, String table) {
+ this.database = database;
+ this.name = table;
+ }
+
+ public void addCol(String column) {
+ colList.add(column);
+ }
+
+ public void setParserList(List<ColumnParser> parserList) {
+ this.parserList = parserList;
+ }
+
+ public void setRawDataTypeList(List<String> rawDataTypeList) {
+ this.rawDataTypeList = rawDataTypeList;
+ }
+
+ public void addParser(ColumnParser columnParser) {
+ parserList.add(columnParser);
+ }
+
+ public void addRawDataType(String rawDataType){
+ this.rawDataTypeList.add(rawDataType);
+ }
+
+ public List<String> getColList() {
+ return colList;
+ }
+
+ public List<String> getRawDataTypeList() {
+ return rawDataTypeList;
+ }
+
+ public String getDatabase() {
+ return database;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List<ColumnParser> getParserList() {
+ return parserList;
+ }
+
+ public List<Object> getDataList() {
+ return dataList;
+ }
+
+ public void setDataList(List<Object> dataList) {
+ this.dataList = dataList;
+ }
+
+ public void setColList(List<String> colList) {
+ this.colList = colList;
+ }
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/BigIntColumnParser.java
similarity index 52%
copy from src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
copy to src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/BigIntColumnParser.java
index adb1c62..610f07d 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/BigIntColumnParser.java
@@ -1,5 +1,3 @@
-
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,12 +15,36 @@
* limitations under the License.
*/
-package org.apache.rocketmq.connect.jdbc.connector;
+package org.apache.rocketmq.connect.jdbc.schema.column;
-import io.openmessaging.connector.api.source.SourceTask;
+import java.math.BigInteger;
-public abstract class JdbcSourceTask extends SourceTask {
-/*
- * To Be Continued
- */
-}
\ No newline at end of file
+public class BigIntColumnParser extends ColumnParser {
+
+ private static BigInteger max = BigInteger.ONE.shiftLeft(64);
+
+ private boolean signed;
+
+ public BigIntColumnParser(String colType) {
+ this.signed = !colType.matches(".* unsigned$");
+ }
+
+ @Override
+ public Object getValue(Object value) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof BigInteger) {
+ return value;
+ }
+
+ Long l = (Long) value;
+ if (!signed && l < 0) {
+ return max.add(BigInteger.valueOf(l));
+ } else {
+ return l;
+ }
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnParser.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnParser.java
new file mode 100644
index 0000000..341064e
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnParser.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.schema.column;
+
+import io.openmessaging.connector.api.data.FieldType;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public abstract class ColumnParser {
+
+ public static ColumnParser getColumnParser(String dataType, String colType, String charset) {
+
+ switch (dataType) {
+ case "tinyint":
+ case "smallint":
+ case "mediumint":
+ case "int":
+ return new IntColumnParser(dataType, colType);
+ case "bigint":
+ return new BigIntColumnParser(colType);
+ case "tinytext":
+ case "text":
+ case "mediumtext":
+ case "longtext":
+ case "varchar":
+ case "char":
+ return new StringColumnParser(charset);
+ case "date":
+ case "datetime":
+ case "timestamp":
+ return new DateTimeColumnParser();
+ case "time":
+ return new TimeColumnParser();
+ case "year":
+ return new YearColumnParser();
+ case "enum":
+ return new EnumColumnParser(colType);
+ case "set":
+ return new SetColumnParser(colType);
+ default:
+ return new DefaultColumnParser();
+ }
+ }
+
+ public static FieldType mapConnectorFieldType(String dataType) {
+
+ switch (dataType) {
+ case "tinyint":
+ case "smallint":
+ case "mediumint":
+ case "int":
+ return FieldType.INT32;
+ case "bigint":
+ return FieldType.BIG_INTEGER;
+ case "tinytext":
+ case "text":
+ case "mediumtext":
+ case "longtext":
+ case "varchar":
+ case "char":
+ return FieldType.STRING;
+ case "date":
+ case "datetime":
+ case "timestamp":
+ case "time":
+ case "year":
+ return FieldType.DATETIME;
+ case "enum":
+ return null;
+ case "set":
+ return null;
+ default:
+ return FieldType.BYTES;
+ }
+ }
+
+ public static String[] extractEnumValues(String colType) {
+ String[] enumValues = {};
+ Matcher matcher = Pattern.compile("(enum|set)\\((.*)\\)").matcher(colType);
+ if (matcher.matches()) {
+ enumValues = matcher.group(2).replace("'", "").split(",");
+ }
+
+ return enumValues;
+ }
+
+ public abstract Object getValue(Object value);
+
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java
new file mode 100644
index 0000000..8736280
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.schema.column;
+
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.TimeZone;
+
+public class DateTimeColumnParser extends ColumnParser {
+
+ private static SimpleDateFormat dateTimeFormat;
+ private static SimpleDateFormat dateTimeUtcFormat;
+
+ static {
+ dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeUtcFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeUtcFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ @Override
+ public Object getValue(Object value) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof Timestamp) {
+ return dateTimeFormat.format(value);
+ }
+
+ if (value instanceof Long) {
+ return dateTimeUtcFormat.format(new Date((Long) value));
+ }
+
+ return value;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DefaultColumnParser.java
similarity index 65%
copy from src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
copy to src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DefaultColumnParser.java
index adb1c62..ee3075a 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DefaultColumnParser.java
@@ -1,5 +1,3 @@
-
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,12 +15,23 @@
* limitations under the License.
*/
-package org.apache.rocketmq.connect.jdbc.connector;
+package org.apache.rocketmq.connect.jdbc.schema.column;
-import io.openmessaging.connector.api.source.SourceTask;
+import org.apache.commons.codec.binary.Base64;
-public abstract class JdbcSourceTask extends SourceTask {
-/*
- * To Be Continued
- */
-}
\ No newline at end of file
+public class DefaultColumnParser extends ColumnParser {
+
+ @Override
+ public Object getValue(Object value) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof byte[]) {
+ return Base64.encodeBase64String((byte[]) value);
+ }
+
+ return value;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/EnumColumnParser.java
similarity index 57%
copy from src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
copy to src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/EnumColumnParser.java
index adb1c62..0fd14ba 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/EnumColumnParser.java
@@ -1,5 +1,3 @@
-
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,12 +15,32 @@
* limitations under the License.
*/
-package org.apache.rocketmq.connect.jdbc.connector;
+package org.apache.rocketmq.connect.jdbc.schema.column;
-import io.openmessaging.connector.api.source.SourceTask;
+public class EnumColumnParser extends ColumnParser {
-public abstract class JdbcSourceTask extends SourceTask {
-/*
- * To Be Continued
- */
-}
\ No newline at end of file
+ private String[] enumValues;
+
+ public EnumColumnParser(String colType) {
+ enumValues = extractEnumValues(colType);
+ }
+
+ @Override
+ public Object getValue(Object value) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof String) {
+ return value;
+ }
+
+ Integer i = (Integer) value;
+ if (i == 0) {
+ return null;
+ } else {
+ return enumValues[i - 1];
+ }
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/IntColumnParser.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/IntColumnParser.java
new file mode 100644
index 0000000..36c6078
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/IntColumnParser.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.schema.column;
+
+public class IntColumnParser extends ColumnParser {
+
+ private int bits;
+ private boolean signed;
+
+ public IntColumnParser(String dataType, String colType) {
+
+ switch (dataType) {
+ case "tinyint":
+ bits = 8;
+ break;
+ case "smallint":
+ bits = 16;
+ break;
+ case "mediumint":
+ bits = 24;
+ break;
+ case "int":
+ bits = 32;
+ }
+
+ this.signed = !colType.matches(".* unsigned$");
+ }
+
+ @Override
+ public Object getValue(Object value) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof Long) {
+ return value;
+ }
+
+ if (value instanceof Integer) {
+ Integer i = (Integer) value;
+ if (signed || i > 0) {
+ return i;
+ } else {
+ return (1L << bits) + i;
+ }
+ }
+
+ return value;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/SetColumnParser.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/SetColumnParser.java
new file mode 100644
index 0000000..d1e6bbc
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/SetColumnParser.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.schema.column;
+
+public class SetColumnParser extends ColumnParser {
+
+ private String[] enumValues;
+
+ public SetColumnParser(String colType) {
+ enumValues = extractEnumValues(colType);
+ }
+
+ @Override
+ public Object getValue(Object value) {
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof String) {
+ return value;
+ }
+
+ StringBuilder builder = new StringBuilder();
+ long l = (Long) value;
+
+ boolean needSplit = false;
+ for (int i = 0; i < enumValues.length; i++) {
+ if (((l >> i) & 1) == 1) {
+ if (needSplit)
+ builder.append(",");
+
+ builder.append(enumValues[i]);
+ needSplit = true;
+ }
+ }
+
+ return builder.toString();
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/StringColumnParser.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/StringColumnParser.java
new file mode 100644
index 0000000..cd4f04f
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/StringColumnParser.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.schema.column;
+
+import org.apache.commons.codec.Charsets;
+
+public class StringColumnParser extends ColumnParser {
+
+ private String charset;
+
+ public StringColumnParser(String charset) {
+ this.charset = charset.toLowerCase();
+ }
+
+ @Override
+ public Object getValue(Object value) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof String) {
+ return value;
+ }
+
+ byte[] bytes = (byte[]) value;
+
+ switch (charset) {
+ case "utf8":
+ case "utf8mb4":
+ return new String(bytes, Charsets.UTF_8);
+ case "latin1":
+ case "ascii":
+ return new String(bytes, Charsets.ISO_8859_1);
+ case "ucs2":
+ return new String(bytes, Charsets.UTF_16);
+ default:
+ return new String(bytes, Charsets.toCharset(charset));
+
+ }
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/TimeColumnParser.java
similarity index 65%
copy from src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
copy to src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/TimeColumnParser.java
index adb1c62..9926d81 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/TimeColumnParser.java
@@ -1,5 +1,3 @@
-
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,12 +15,25 @@
* limitations under the License.
*/
-package org.apache.rocketmq.connect.jdbc.connector;
+package org.apache.rocketmq.connect.jdbc.schema.column;
-import io.openmessaging.connector.api.source.SourceTask;
+import java.sql.Time;
+import java.sql.Timestamp;
-public abstract class JdbcSourceTask extends SourceTask {
-/*
- * To Be Continued
- */
-}
\ No newline at end of file
+public class TimeColumnParser extends ColumnParser {
+
+ @Override
+ public Object getValue(Object value) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof Timestamp) {
+
+ return new Time(((Timestamp) value).getTime());
+ }
+
+ return value;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/YearColumnParser.java
similarity index 61%
copy from src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
copy to src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/YearColumnParser.java
index adb1c62..14cc798 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/YearColumnParser.java
@@ -1,5 +1,3 @@
-
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,12 +15,26 @@
* limitations under the License.
*/
-package org.apache.rocketmq.connect.jdbc.connector;
+package org.apache.rocketmq.connect.jdbc.schema.column;
-import io.openmessaging.connector.api.source.SourceTask;
+import java.sql.Date;
+import java.util.Calendar;
-public abstract class JdbcSourceTask extends SourceTask {
-/*
- * To Be Continued
- */
-}
\ No newline at end of file
+public class YearColumnParser extends ColumnParser {
+
+ @Override
+ public Object getValue(Object value) {
+
+ if (value == null) {
+ return null;
+ }
+
+ if (value instanceof Date) {
+ Calendar calendar = Calendar.getInstance();
+ calendar.setTime((Date) value);
+ return calendar.get(Calendar.YEAR);
+ }
+
+ return value;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
new file mode 100644
index 0000000..61323d4
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -0,0 +1,187 @@
+package org.apache.rocketmq.connect.jdbc.source;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import javax.sql.DataSource;
+
+import org.apache.rocketmq.connect.jdbc.Config;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alibaba.druid.pool.DruidDataSourceFactory;
+
+
+import org.apache.rocketmq.connect.jdbc.schema.*;
+import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
+
+public class Querier {
+ static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
+ private final Logger log = LoggerFactory.getLogger(getClass()); // use concrete subclass
+ protected String topicPrefix;
+ protected String jdbcUrl;
+ private final Queue<Connection> connections = new ConcurrentLinkedQueue<>();
+ private Config config = new Config();
+ private DataSource dataSource;
+ private List<Table> list=new LinkedList<>();
+
+
+
+ public List<Table> getList() {
+ return list;
+ }
+
+ public void setList(List<Table> list) {
+ this.list = list;
+ }
+
+ public Connection getConnection() throws SQLException {
+
+ // These config names are the same for both source and sink configs ...
+ String username = config.jdbcUsername;
+ String dbPassword = config.jdbcPassword;
+ jdbcUrl = config.jdbcUrl;
+ Properties properties = new Properties();
+ if (username != null) {
+ properties.setProperty("user", username);
+ }
+ if (dbPassword != null) {
+ properties.setProperty("password", dbPassword);
+ }
+ Connection connection = DriverManager.getConnection(jdbcUrl, properties);
+
+ connections.add(connection);
+ return connection;
+ }
+
+ public void close() {
+ Connection conn;
+ while ((conn = connections.poll()) != null) {
+ try {
+ conn.close();
+ } catch (Throwable e) {
+ log.warn("Error while closing connection to {}", "jdbc", e);
+ }
+ }
+ }
+
+ protected PreparedStatement createDBPreparedStatement(Connection db) throws SQLException {
+
+ String SQL = "select table_name,column_name,data_type,column_type,character_set_name "
+ + "from information_schema.columns " + "where table_schema = jdbc_db order by ORDINAL_POSITION";
+
+ log.trace("Creating a PreparedStatement '{}'", SQL);
+ PreparedStatement stmt = db.prepareStatement(SQL);
+ return stmt;
+
+ }
+
+ protected PreparedStatement createPreparedStatement(Connection db, String string) throws SQLException {
+ String query = "select * from " + string;
+ log.trace("Creating a PreparedStatement '{}'", query);
+ PreparedStatement stmt = db.prepareStatement(query);
+ return stmt;
+
+ }
+
+ protected ResultSet executeQuery(PreparedStatement stmt) throws SQLException {
+ return stmt.executeQuery();
+ }
+
+ public static void main(String[] args) throws Exception {
+ Querier querier = new Querier();
+ try {
+ querier.start();
+ querier.poll();
+
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ }
+
+ private Schema schema;
+
+ private Map<Long, Table> tableMap = new HashMap<>();
+
+ public void poll() {
+ try {
+
+ PreparedStatement stmt;
+ String query = "select * from ";
+ Connection conn = dataSource.getConnection();
+
+ for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
+ String db = entry.getKey();
+ if(!db.contains("jdbc_db"))
+ continue;
+ Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<String, Table> tableEntry = iterator.next();
+ String tb=tableEntry.getKey();
+ stmt = conn.prepareStatement(query+db + "." +tb);
+ ResultSet rs;
+ rs = stmt.executeQuery();
+ List<String> colList = tableEntry.getValue().getColList();
+ List<String> DataTypeList = tableEntry.getValue().getRawDataTypeList();
+ List<ColumnParser> ParserList = tableEntry.getValue().getParserList();
+
+ while(rs.next()) {
+ Table table=new Table(db, tb);
+ System.out.print("|");
+ table.setColList(colList);
+ table.setRawDataTypeList(DataTypeList);
+ table.setParserList(ParserList);
+
+ for (String string : colList) {
+ table.getDataList().add(rs.getObject(string));
+ System.out.print(string+" : "+rs.getObject(string)+"|");
+ }
+ list.add(table);
+ System.out.println();
+ }
+ }
+ }
+
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ public void start() throws Exception {
+ initDataSource();
+ schema = new Schema(dataSource);
+ schema.load();
+ }
+
+ private void initDataSource() throws Exception {
+ Map<String, String> map = new HashMap<>();
+ map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
+ map.put("url",
+ "jdbc:mysql://" + config.jdbcUrl + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
+ map.put("username", config.jdbcUsername);
+ map.put("password", config.jdbcPassword);
+ map.put("initialSize", "2");
+ map.put("maxActive", "2");
+ map.put("maxWait", "60000");
+ map.put("timeBetweenEvictionRunsMillis", "60000");
+ map.put("minEvictableIdleTimeMillis", "300000");
+ map.put("validationQuery", "SELECT 1 FROM DUAL");
+ map.put("testWhileIdle", "true");
+ dataSource = DruidDataSourceFactory.createDataSource(map);
+ }
+
+}
[rocketmq-connect] 12/43: Delete ReplicatorTest.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit a1d120018304316e170a37d9fd2c7506a977ab68
Author: Yuchen Li <yu...@126.com>
AuthorDate: Fri Aug 2 00:31:12 2019 +0800
Delete ReplicatorTest.java
---
.../rocketmq/connect/jdbc/ReplicatorTest.java | 74 ----------------------
1 file changed, 74 deletions(-)
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/ReplicatorTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/ReplicatorTest.java
deleted file mode 100644
index 88d5586..0000000
--- a/src/test/java/org/apache/rocketmq/connect/jdbc/ReplicatorTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-///*
-// * Licensed to the Apache Software Foundation (ASF) under one or more
-// * contributor license agreements. See the NOTICE file distributed with
-// * this work for additional information regarding copyright ownership.
-// * The ASF licenses this file to You under the Apache License, Version 2.0
-// * (the "License"); you may not use this file except in compliance with
-// * the License. You may obtain a copy of the License at
-// *
-// * http://www.apache.org/licenses/LICENSE-2.0
-// *
-// * Unless required by applicable law or agreed to in writing, software
-// * distributed under the License is distributed on an "AS IS" BASIS,
-// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// * See the License for the specific language governing permissions and
-// * limitations under the License.
-// */
-//
-//package org.apache.rocketmq.connect.jms;
-//
-//import java.lang.reflect.Field;
-//
-//import javax.jms.Message;
-//
-//import org.apache.activemq.command.ActiveMQTextMessage;
-//import org.apache.rocketmq.connect.jms.pattern.PatternProcessor;
-//import org.junit.Before;
-//import org.junit.Test;
-//import org.mockito.Mockito;
-//
-//import org.junit.Assert;
-//
-//public class ReplicatorTest {
-//
-// Replicator replicator;
-//
-// PatternProcessor patternProcessor;
-//
-// Config config;
-//
-// @Before
-// public void before() throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException {
-// config = new Config();
-// replicator = new Replicator(config,null);
-//
-// patternProcessor = Mockito.mock(PatternProcessor.class);
-//
-// Field processor = Replicator.class.getDeclaredField("processor");
-// processor.setAccessible(true);
-// processor.set(replicator, patternProcessor);
-// }
-//
-// @Test(expected = RuntimeException.class)
-// public void startTest() throws Exception {
-// replicator.start();
-// }
-//
-// @Test
-// public void stop() throws Exception {
-// replicator.stop();
-// Mockito.verify(patternProcessor, Mockito.times(1)).stop();
-// }
-//
-// @Test
-// public void commitAddGetQueueTest() {
-// Message message = new ActiveMQTextMessage();
-// replicator.commit(message, false);
-// Assert.assertEquals(replicator.getQueue().poll(), message);
-// }
-//
-// @Test
-// public void getConfigTest() {
-// Assert.assertEquals(replicator.getConfig(), config);
-// }
-//}
[rocketmq-connect] 29/43: [ISSUE #489] JDBC Connector support divide task by topic strategy (#490)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 341d6f356047b0fe908a2aad1ca901cdd7317aef
Author: Xiongmengfei <Xi...@163.com>
AuthorDate: Tue Dec 31 15:21:39 2019 +0800
[ISSUE #489] JDBC Connector support divide task by topic strategy (#490)
* [ISSUE #489] JDBC Connector support divide task by topic strategy
* [ISSUE #489] JDBC Connector support divide task by topic strategy
---
.../rocketmq/connect/jdbc/config/Config.java | 70 +++++++++----
.../rocketmq/connect/jdbc/config/DataType.java | 26 +++++
.../connect/jdbc/config/DbConnectorConfig.java | 84 ++++++++++++++++
.../connect/jdbc/config/SinkDbConnectorConfig.java | 67 ++++++++++++
.../jdbc/config/SourceDbConnectorConfig.java | 73 ++++++++++++++
.../connect/jdbc/config/TaskDivideConfig.java | 112 +++++++++++++++++++++
.../connect/jdbc/config/TaskTopicInfo.java | 37 +++++++
.../connect/jdbc/connector/JdbcSinkConnector.java | 39 +++++--
.../connect/jdbc/connector/JdbcSinkTask.java | 1 +
.../jdbc/connector/JdbcSourceConnector.java | 37 +++++--
.../connect/jdbc/connector/JdbcSourceTask.java | 13 +--
.../connect/jdbc/strategy/DivideStrategyEnum.java | 23 +++++
.../jdbc/strategy/DivideTaskByConsistentHash.java | 82 +++++++++++++++
.../connect/jdbc/strategy/DivideTaskByQueue.java | 62 ++++++++++++
.../connect/jdbc/strategy/DivideTaskByTopic.java | 104 +++++++++++++++++++
.../connect/jdbc/strategy/TaskDivideStrategy.java | 32 ++++++
16 files changed, 822 insertions(+), 40 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index 5491d43..91a3e51 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.connect.jdbc.config;
+import com.alibaba.fastjson.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,9 +29,11 @@ public class Config {
private static final Logger LOG = LoggerFactory.getLogger(Config.class);
/* Database Connection Config */
- private String jdbcUrl;
- private String jdbcUsername;
- private String jdbcPassword;
+ private String dbUrl;
+ private String dbPort;
+ private String dbUsername;
+ private String dbPassword;
+ private String dataType;
private String rocketmqTopic;
private String jdbcBackoff;
private String jdbcAttempts;
@@ -44,6 +47,18 @@ public class Config {
private String whiteDataBase;
private String whiteTable;
+ public static final String CONN_TASK_PARALLELISM = "task-parallelism";
+ public static final String CONN_TASK_DIVIDE_STRATEGY = "task-divide-strategy";
+ public static final String CONN_WHITE_LIST = "whiteDataBase";
+ public static final String CONN_SOURCE_RECORD_CONVERTER = "source-record-converter";
+ public static final String CONN_DB_IP = "dbUrl";
+ public static final String CONN_DB_PORT = "dbPort";
+ public static final String CONN_DB_USERNAME = "dbUsername";
+ public static final String CONN_DB_PASSWORD = "dbPassword";
+ public static final String CONN_DATA_TYPE = "dataType";
+ public static final String CONN_TOPIC_NAMES = "topicNames";
+ public static final String CONN_DB_MODE = "mode";
+
/* Mode Config */
private String mode = "";
private String incrementingColumnName = "";
@@ -57,15 +72,16 @@ public class Config {
private int batchMaxRows = 100;
private long tablePollInterval = 60000;
private long timestampDelayInterval = 0;
- private String dbTimezone = "UTC";
+ private String dbTimezone = "GMT+8";
private String queueName;
private Logger log = LoggerFactory.getLogger(Config.class);
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
{
- add("jdbcUrl");
- add("jdbcUsername");
- add("jdbcPassword");
+ add("dbUrl");
+ add("dbPort");
+ add("dbUsername");
+ add("dbPassword");
add("mode");
add("rocketmqTopic");
}
@@ -79,28 +95,44 @@ public class Config {
this.queueName = queueName;
}
- public String getJdbcUrl() {
- return jdbcUrl;
+ public String getDbUrl() {
+ return dbUrl;
+ }
+
+ public void setDbUrl(String dbUrl) {
+ this.dbUrl = dbUrl;
+ }
+
+ public String getDbPort() {
+ return dbPort;
+ }
+
+ public void setDbPort(String dbPort) {
+ this.dbPort = dbPort;
+ }
+
+ public String getDbUsername() {
+ return dbUsername;
}
- public void setJdbcUrl(String jdbcUrl) {
- this.jdbcUrl = jdbcUrl;
+ public void setDbUsername(String dbUsername) {
+ this.dbUsername = dbUsername;
}
- public String getJdbcUsername() {
- return jdbcUsername;
+ public String getDbPassword() {
+ return dbPassword;
}
- public void setJdbcUsername(String jdbcUsername) {
- this.jdbcUsername = jdbcUsername;
+ public void setDbPassword(String dbPassword) {
+ this.dbPassword = dbPassword;
}
- public String getJdbcPassword() {
- return jdbcPassword;
+ public String getDataType() {
+ return dataType;
}
- public void setJdbcPassword(String jdbcPassword) {
- this.jdbcPassword = jdbcPassword;
+ public void setDataType(String dataType) {
+ this.dataType = dataType;
}
public String getRocketmqTopic() {
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/DataType.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/DataType.java
new file mode 100644
index 0000000..ef7408a
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/DataType.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.config;
+
+public enum DataType {
+
+ COMMON_MESSAGE,
+ TOPIC_CONFIG,
+ BROKER_CONFIG,
+ SUB_CONFIG,
+ OFFSET
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/DbConnectorConfig.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/DbConnectorConfig.java
new file mode 100644
index 0000000..43bd165
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/DbConnectorConfig.java
@@ -0,0 +1,84 @@
+package org.apache.rocketmq.connect.jdbc.config;
+
+import io.openmessaging.KeyValue;
+import org.apache.rocketmq.connect.jdbc.strategy.TaskDivideStrategy;
+
+public abstract class DbConnectorConfig {
+
+ public TaskDivideStrategy taskDivideStrategy;
+ public String dbUrl;
+ public String dbPort;
+ public String dbUserName;
+ public String dbPassword;
+ public String converter;
+ public int taskParallelism;
+ public String mode;
+
+ public abstract void validate(KeyValue config);
+
+ public abstract <T> T getWhiteTopics();
+
+ public TaskDivideStrategy getTaskDivideStrategy() {
+ return taskDivideStrategy;
+ }
+
+ public void setTaskDivideStrategy(TaskDivideStrategy taskDivideStrategy) {
+ this.taskDivideStrategy = taskDivideStrategy;
+ }
+
+ public String getDbUrl() {
+ return dbUrl;
+ }
+
+ public void setDbUrl(String dbUrl) {
+ this.dbUrl = dbUrl;
+ }
+
+ public String getDbPort() {
+ return dbPort;
+ }
+
+ public void setDbPort(String dbPort) {
+ this.dbPort = dbPort;
+ }
+
+ public String getDbUserName() {
+ return dbUserName;
+ }
+
+ public void setDbUserName(String dbUserName) {
+ this.dbUserName = dbUserName;
+ }
+
+ public String getDbPassword() {
+ return dbPassword;
+ }
+
+ public void setDbPassword(String dbPassword) {
+ this.dbPassword = dbPassword;
+ }
+
+ public String getConverter() {
+ return converter;
+ }
+
+ public void setConverter(String converter) {
+ this.converter = converter;
+ }
+
+ public int getTaskParallelism() {
+ return taskParallelism;
+ }
+
+ public void setTaskParallelism(int taskParallelism) {
+ this.taskParallelism = taskParallelism;
+ }
+
+ public String getMode() {
+ return mode;
+ }
+
+ public void setMode(String mode) {
+ this.mode = mode;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
new file mode 100644
index 0000000..851b253
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
@@ -0,0 +1,67 @@
+package org.apache.rocketmq.connect.jdbc.config;
+
+import io.openmessaging.KeyValue;
+import org.apache.rocketmq.connect.jdbc.strategy.DivideStrategyEnum;
+import org.apache.rocketmq.connect.jdbc.strategy.DivideTaskByQueue;
+import org.apache.rocketmq.connect.jdbc.strategy.DivideTaskByTopic;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class SinkDbConnectorConfig extends DbConnectorConfig{
+
+ private Set<String> whiteList;
+
+ public SinkDbConnectorConfig(){
+ }
+
+ @Override
+ public void validate(KeyValue config) {
+ this.taskParallelism = config.getInt(Config.CONN_TASK_PARALLELISM, 0);
+
+ int strategy = config.getInt(Config.CONN_TASK_DIVIDE_STRATEGY, DivideStrategyEnum.BY_QUEUE.ordinal());
+ if (strategy == DivideStrategyEnum.BY_QUEUE.ordinal()) {
+ this.taskDivideStrategy = new DivideTaskByQueue();
+ } else {
+ this.taskDivideStrategy = new DivideTaskByTopic();
+ }
+
+ buildWhiteList(config);
+
+ this.converter = config.getString(Config.CONN_SOURCE_RECORD_CONVERTER);
+ this.dbUrl = config.getString(Config.CONN_DB_IP);
+ this.dbPort = config.getString(Config.CONN_DB_PORT);
+ this.dbUserName = config.getString(Config.CONN_DB_USERNAME);
+ this.dbPassword = config.getString(Config.CONN_DB_PASSWORD);
+
+ }
+
+ private void buildWhiteList(KeyValue config) {
+ this.whiteList = new HashSet<>();
+ String whiteListStr = config.getString(Config.CONN_WHITE_LIST, "");
+ String[] wl = whiteListStr.trim().split(",");
+ if (wl.length <= 0)
+ throw new IllegalArgumentException("White list must be not empty.");
+ else {
+ this.whiteList.clear();
+ for (String t : wl) {
+ this.whiteList.add(t.trim());
+ }
+ }
+ }
+
+
+ public Set<String> getWhiteList() {
+ return whiteList;
+ }
+
+ public void setWhiteList(Set<String> whiteList) {
+ this.whiteList = whiteList;
+ }
+
+ @Override
+ public Set<String> getWhiteTopics() {
+ return getWhiteList();
+ }
+
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java
new file mode 100644
index 0000000..801e411
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java
@@ -0,0 +1,73 @@
+package org.apache.rocketmq.connect.jdbc.config;
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import org.apache.rocketmq.connect.jdbc.strategy.DivideStrategyEnum;
+import org.apache.rocketmq.connect.jdbc.strategy.DivideTaskByQueue;
+import org.apache.rocketmq.connect.jdbc.strategy.DivideTaskByTopic;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class SourceDbConnectorConfig extends DbConnectorConfig{
+
+ private Map<String, String> whiteMap;
+
+ public SourceDbConnectorConfig(){
+ }
+
+ @Override
+ public void validate(KeyValue config) {
+ this.taskParallelism = config.getInt(Config.CONN_TASK_PARALLELISM, 0);
+
+ int strategy = config.getInt(Config.CONN_TASK_DIVIDE_STRATEGY, DivideStrategyEnum.BY_QUEUE.ordinal());
+ if (strategy == DivideStrategyEnum.BY_QUEUE.ordinal()) {
+ this.taskDivideStrategy = new DivideTaskByQueue();
+ } else {
+ this.taskDivideStrategy = new DivideTaskByTopic();
+ }
+
+ buildWhiteMap(config);
+
+ this.converter = config.getString(Config.CONN_SOURCE_RECORD_CONVERTER);
+ this.dbUrl = config.getString(Config.CONN_DB_IP);
+ this.dbPort = config.getString(Config.CONN_DB_PORT);
+ this.dbUserName = config.getString(Config.CONN_DB_USERNAME);
+ this.dbPassword = config.getString(Config.CONN_DB_PASSWORD);
+ this.mode = config.getString(Config.CONN_DB_MODE, "bulk");
+
+ }
+
+ private void buildWhiteMap(KeyValue config) {
+ this.whiteMap = new HashMap<>(16);
+ String whiteListStr = config.getString(Config.CONN_WHITE_LIST, "");
+ JSONObject whiteDataBaseObject = JSONObject.parseObject(whiteListStr);
+ if(whiteDataBaseObject.keySet().size() <= 0){
+ throw new IllegalArgumentException("white data base must be not empty.");
+ }else {
+ this.whiteMap.clear();
+ for (String dbName : whiteDataBaseObject.keySet()){
+ JSONObject whiteTableObject = (JSONObject) whiteDataBaseObject.get(dbName);
+ for (String tableName : whiteTableObject.keySet()){
+ String dbTableKey = dbName + "-" + tableName;
+ this.whiteMap.put(dbTableKey, whiteTableObject.getString(tableName));
+ }
+ }
+ }
+ }
+
+
+ public Map<String, String> getWhiteMap() {
+ return whiteMap;
+ }
+
+ public void setWhiteMap(Map<String, String> whiteMap) {
+ this.whiteMap = whiteMap;
+ }
+
+ @Override
+ public Map<String, String> getWhiteTopics() {
+ return getWhiteMap();
+ }
+
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskDivideConfig.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskDivideConfig.java
new file mode 100644
index 0000000..8b15a2f
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskDivideConfig.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.config;
+
+public class TaskDivideConfig {
+
+ private String dbUrl;
+
+ private String dbPort;
+
+ private String dbUserName;
+
+ private String dbPassword;
+
+ private String srcRecordConverter;
+
+ private int dataType;
+
+ private int taskParallelism;
+
+ private String mode;
+
+ public TaskDivideConfig(String dbUrl, String dbPort, String dbUserName, String dbPassword, String srcRecordConverter,
+ int dataType, int taskParallelism, String mode) {
+ this.dbUrl = dbUrl;
+ this.dbPort = dbPort;
+ this.dbUserName = dbUserName;
+ this.dbPassword = dbPassword;
+ this.srcRecordConverter = srcRecordConverter;
+ this.dataType = dataType;
+ this.taskParallelism = taskParallelism;
+ this.mode = mode;
+ }
+
+ public String getDbUrl() {
+ return dbUrl;
+ }
+
+ public void setDbUrl(String dbUrl) {
+ this.dbUrl = dbUrl;
+ }
+
+ public String getDbPort() {
+ return dbPort;
+ }
+
+ public void setDbPort(String dbPort) {
+ this.dbPort = dbPort;
+ }
+
+ public String getDbUserName() {
+ return dbUserName;
+ }
+
+ public void setDbUserName(String dbUserName) {
+ this.dbUserName = dbUserName;
+ }
+
+ public String getDbPassword() {
+ return dbPassword;
+ }
+
+ public void setDbPassword(String dbPassword) {
+ this.dbPassword = dbPassword;
+ }
+
+ public String getSrcRecordConverter() {
+ return srcRecordConverter;
+ }
+
+ public void setSrcRecordConverter(String srcRecordConverter) {
+ this.srcRecordConverter = srcRecordConverter;
+ }
+
+ public int getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(int dataType) {
+ this.dataType = dataType;
+ }
+
+ public int getTaskParallelism() {
+ return taskParallelism;
+ }
+
+ public void setTaskParallelism(int taskParallelism) {
+ this.taskParallelism = taskParallelism;
+ }
+
+ public String getMode() {
+ return mode;
+ }
+
+ public void setMode(String mode) {
+ this.mode = mode;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskTopicInfo.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskTopicInfo.java
new file mode 100644
index 0000000..5c2a21e
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskTopicInfo.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.config;
+
+import org.apache.rocketmq.common.message.MessageQueue;
+
+public class TaskTopicInfo extends MessageQueue {
+
+ private String targetTopic;
+
+ public TaskTopicInfo(String sourceTopic, String brokerName, int queueId, String targetTopic) {
+ super(sourceTopic, brokerName, queueId);
+ this.targetTopic = targetTopic;
+ }
+
+ public String getTargetTopic() {
+ return this.targetTopic;
+ }
+
+ public void setTargetTopic(String targetTopic) {
+ this.targetTopic = targetTopic;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
index e1d1256..935ad52 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
@@ -3,15 +3,23 @@ package org.apache.rocketmq.connect.jdbc.connector;
import io.openmessaging.KeyValue;
import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.sink.SinkConnector;
-import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.config.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
public class JdbcSinkConnector extends SinkConnector {
-
+ private static final Logger log = LoggerFactory.getLogger(JdbcSinkConnector.class);
private KeyValue config;
+ private DbConnectorConfig dbConnectorConfig;
+ private volatile boolean configValid = false;
+
+ public JdbcSinkConnector(){
+ dbConnectorConfig = new SinkDbConnectorConfig();
+ }
@Override
public String verifyAndSetConfig(KeyValue config) {
@@ -20,7 +28,13 @@ public class JdbcSinkConnector extends SinkConnector {
return "Request config key: " + requestKey;
}
}
- this.config = config;
+ try {
+ this.dbConnectorConfig.validate(config);
+ } catch (IllegalArgumentException e) {
+ return e.getMessage();
+ }
+ this.configValid = true;
+
return "";
}
@@ -51,8 +65,21 @@ public class JdbcSinkConnector extends SinkConnector {
@Override
public List<KeyValue> taskConfigs() {
- List<KeyValue> config = new ArrayList<>();
- config.add(this.config);
- return config;
+ log.info("List.start");
+ if (!configValid) {
+ return new ArrayList<KeyValue>();
+ }
+
+ TaskDivideConfig tdc = new TaskDivideConfig(
+ this.dbConnectorConfig.getDbUrl(),
+ this.dbConnectorConfig.getDbPort(),
+ this.dbConnectorConfig.getDbUserName(),
+ this.dbConnectorConfig.getDbPassword(),
+ this.dbConnectorConfig.getConverter(),
+ DataType.COMMON_MESSAGE.ordinal(),
+ this.dbConnectorConfig.getTaskParallelism(),
+ this.dbConnectorConfig.getMode()
+ );
+ return this.dbConnectorConfig.getTaskDivideStrategy().divide(this.dbConnectorConfig, tdc);
}
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java
index 4b55e5a..31f43e3 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java
@@ -114,6 +114,7 @@ public class JdbcSinkTask extends SinkTask {
try {
if (connection != null){
connection.close();
+ log.info("jdbc sink task connection is closed.");
}
} catch (Throwable e) {
log.warn("sink task stop error while closing connection to {}", "jdbc", e);
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
index 796f0e6..a083e84 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
@@ -20,7 +20,7 @@ package org.apache.rocketmq.connect.jdbc.connector;
import java.util.ArrayList;
import java.util.List;
-import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.config.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,19 +30,29 @@ import io.openmessaging.connector.api.source.SourceConnector;
public class JdbcSourceConnector extends SourceConnector {
private static final Logger log = LoggerFactory.getLogger(JdbcSourceConnector.class);
- private KeyValue config;
+ private DbConnectorConfig dbConnectorConfig;
+ private volatile boolean configValid = false;
+
+ public JdbcSourceConnector() {
+ dbConnectorConfig = new SourceDbConnectorConfig();
+ }
@Override
public String verifyAndSetConfig(KeyValue config) {
- log.info("1216123 JdbcSourceConnector verifyAndSetConfig enter");
+ log.info("JdbcSourceConnector verifyAndSetConfig enter");
for (String requestKey : Config.REQUEST_CONFIG) {
if (!config.containsKey(requestKey)) {
return "Request config key: " + requestKey;
}
}
- this.config = config;
+ try {
+ this.dbConnectorConfig.validate(config);
+ } catch (IllegalArgumentException e) {
+ return e.getMessage();
+ }
+ this.configValid = true;
return "";
}
@@ -75,8 +85,21 @@ public class JdbcSourceConnector extends SourceConnector {
@Override
public List<KeyValue> taskConfigs() {
log.info("List.start");
- List<KeyValue> config = new ArrayList<>();
- config.add(this.config);
- return config;
+ if (!configValid) {
+ return new ArrayList<KeyValue>();
+ }
+
+ TaskDivideConfig tdc = new TaskDivideConfig(
+ this.dbConnectorConfig.getDbUrl(),
+ this.dbConnectorConfig.getDbPort(),
+ this.dbConnectorConfig.getDbUserName(),
+ this.dbConnectorConfig.getDbPassword(),
+ this.dbConnectorConfig.getConverter(),
+ DataType.COMMON_MESSAGE.ordinal(),
+ this.dbConnectorConfig.getTaskParallelism(),
+ this.dbConnectorConfig.getMode()
+ );
+ return this.dbConnectorConfig.getTaskDivideStrategy().divide(this.dbConnectorConfig, tdc);
}
+
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index 943d432..d533395 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -22,11 +22,7 @@ import io.openmessaging.connector.api.source.SourceTask;
import java.nio.ByteBuffer;
import java.sql.Connection;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Timer;
+import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -82,7 +78,7 @@ public class JdbcSourceTask extends SourceTask {
querier = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
else
querier = tableQueue.peek();
- Timer timer = new java.util.Timer();
+ Timer timer = new Timer();
try {
Thread.currentThread();
Thread.sleep(1000);//毫秒
@@ -114,7 +110,7 @@ public class JdbcSourceTask extends SourceTask {
}
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap(config.getJdbcUrl().getBytes("UTF-8")),
+ ByteBuffer.wrap((config.getDbUrl() + config.getDbPort()).getBytes("UTF-8")),
ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8")));
res.add(sourceDataEntry);
log.debug("sourceDataEntry : {}", JSONObject.toJSONString(sourceDataEntry));
@@ -163,8 +159,9 @@ public class JdbcSourceTask extends SourceTask {
@Override
public void stop() {
try {
- if (connection != null){
+ if (connection != null) {
connection.close();
+ log.info("jdbc source task connection is closed.");
}
} catch (Throwable e) {
log.warn("source task stop error while closing connection to {}", "jdbc", e);
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideStrategyEnum.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideStrategyEnum.java
new file mode 100644
index 0000000..0afa470
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideStrategyEnum.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.strategy;
+
+public enum DivideStrategyEnum {
+
+ BY_TOPIC,
+ BY_QUEUE
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java
new file mode 100644
index 0000000..bac7358
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java
@@ -0,0 +1,82 @@
+package org.apache.rocketmq.connect.jdbc.strategy;/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.internal.DefaultKeyValue;
+import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter;
+import org.apache.rocketmq.common.consistenthash.Node;
+import org.apache.rocketmq.replicator.config.DataType;
+import org.apache.rocketmq.replicator.config.TaskConfigEnum;
+import org.apache.rocketmq.replicator.config.TaskDivideConfig;
+import org.apache.rocketmq.replicator.config.TaskTopicInfo;
+
+import java.util.*;
+
+public class DivideTaskByConsistentHash extends TaskDivideStrategy {
+ @Override public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
+
+ List<KeyValue> config = new ArrayList<>();
+ int parallelism = tdc.getTaskParallelism();
+ Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
+ int id = -1;
+
+ Collection<ClientNode> cidNodes = new ArrayList<>();
+ for (int i = 0; i < parallelism; i++) {
+ cidNodes.add(new ClientNode(i, Integer.toString(i)));
+ queueTopicList.put(i, new ArrayList<>());
+ }
+
+ ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
+
+ for (String t : topicMap.keySet()) {
+ for (TaskTopicInfo queue : topicMap.get(t)) {
+ ClientNode clientNode = router.routeNode(queue.toString());
+ if (clientNode != null) {
+ queueTopicList.get(clientNode.index).add(queue);
+ }
+ }
+ }
+
+ for (int i = 0; i < parallelism; i++) {
+ KeyValue keyValue = new DefaultKeyValue();
+ keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
+ keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
+ keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
+ config.add(keyValue);
+ }
+
+ return config;
+ }
+
+ private static class ClientNode implements Node {
+ private final String clientID;
+ private final int index;
+
+ public ClientNode(int index, String clientID) {
+ this.index = index;
+ this.clientID = clientID;
+ }
+
+ @Override
+ public String getKey() {
+ return clientID;
+ }
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
new file mode 100644
index 0000000..7ef5c31
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.strategy;
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.internal.DefaultKeyValue;
+import org.apache.rocketmq.replicator.config.DataType;
+import org.apache.rocketmq.replicator.config.TaskConfigEnum;
+import org.apache.rocketmq.replicator.config.TaskDivideConfig;
+import org.apache.rocketmq.replicator.config.TaskTopicInfo;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class DivideTaskByQueue extends TaskDivideStrategy {
+ @Override
+ public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicRouteMap, TaskDivideConfig tdc) {
+
+ List<KeyValue> config = new ArrayList<KeyValue>();
+ int parallelism = tdc.getTaskParallelism();
+ Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<Integer, List<TaskTopicInfo>>();
+ int id = -1;
+ for (String t : topicRouteMap.keySet()) {
+ for (TaskTopicInfo taskTopicInfo : topicRouteMap.get(t)) {
+ int ind = ++id % parallelism;
+ if (!queueTopicList.containsKey(ind)) {
+ queueTopicList.put(ind, new ArrayList<TaskTopicInfo>());
+ }
+ queueTopicList.get(ind).add(taskTopicInfo);
+ }
+ }
+
+ for (int i = 0; i < parallelism; i++) {
+ KeyValue keyValue = new DefaultKeyValue();
+ keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
+ keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
+ keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
+ keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
+ config.add(keyValue);
+ }
+
+ return config;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
new file mode 100644
index 0000000..762c7a0
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.strategy;
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import io.openmessaging.internal.DefaultKeyValue;
+import org.apache.rocketmq.connect.jdbc.config.*;
+import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
+
+import java.util.*;
+
+public class DivideTaskByTopic extends TaskDivideStrategy {
+ @Override
+ public List<KeyValue> divide(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
+ if (dbConnectorConfig instanceof SourceDbConnectorConfig){
+ return divideSourceTaskByTopic(dbConnectorConfig, tdc);
+ }else {
+ return divideSinkTaskByTopic(dbConnectorConfig, tdc);
+ }
+
+ }
+
+ private List<KeyValue> divideSourceTaskByTopic(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
+ List<KeyValue> config = new ArrayList<KeyValue>();
+ int parallelism = tdc.getTaskParallelism();
+ int id = -1;
+ Map<String, String> topicRouteMap = ((SourceDbConnectorConfig)dbConnectorConfig).getWhiteTopics();
+ Map<Integer, Map<String, Map<String, String>>> taskTopicList = new HashMap<>();
+ for (Map.Entry<String, String> entry : topicRouteMap.entrySet()) {
+ int ind = ++id % parallelism;
+ if (!taskTopicList.containsKey(ind)) {
+ taskTopicList.put(ind, new HashMap<>());
+ }
+ String dbKey = entry.getKey().split("-")[0];
+ String tableKey = entry.getKey().split("-")[1];
+ String filter = entry.getValue();
+ Map<String, String> tableMap = new HashMap<>();
+ tableMap.put(tableKey, filter);
+ taskTopicList.get(ind).put(dbKey, tableMap);
+ }
+
+ for (int i = 0; i < parallelism; i++) {
+ KeyValue keyValue = new DefaultKeyValue();
+
+ keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
+ keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
+ keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
+ keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
+ keyValue.put(Config.CONN_WHITE_LIST, JSONObject.toJSONString(taskTopicList.get(i)));
+ keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
+ keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
+ keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
+ config.add(keyValue);
+ }
+
+ return config;
+ }
+
+ private List<KeyValue> divideSinkTaskByTopic(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc) {
+ List<KeyValue> config = new ArrayList<KeyValue>();
+ int parallelism = tdc.getTaskParallelism();
+ int id = -1;
+ Set<String> topicRouteSet = ((SinkDbConnectorConfig)dbConnectorConfig).getWhiteTopics();
+ Map<Integer, String> taskTopicList = new HashMap<>();
+ for (String topicName : topicRouteSet) {
+ int ind = ++id % parallelism;
+ if (!taskTopicList.containsKey(ind)) {
+ taskTopicList.put(ind, topicName);
+ }
+ }
+
+ for (int i = 0; i < parallelism; i++) {
+ KeyValue keyValue = new DefaultKeyValue();
+ keyValue.put(Config.CONN_DB_IP, tdc.getDbUrl());
+ keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
+ keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
+ keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
+ keyValue.put(Config.CONN_TOPIC_NAMES, JSONObject.toJSONString(taskTopicList.get(i)));
+ keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
+ keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
+ keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
+ config.add(keyValue);
+ }
+
+ return config;
+ }
+
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/TaskDivideStrategy.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/TaskDivideStrategy.java
new file mode 100644
index 0000000..736fcac
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/TaskDivideStrategy.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.rocketmq.connect.jdbc.strategy;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import org.apache.rocketmq.connect.jdbc.config.DbConnectorConfig;
+import org.apache.rocketmq.connect.jdbc.config.TaskDivideConfig;
+import org.apache.rocketmq.connect.jdbc.config.TaskTopicInfo;
+
+import java.util.List;
+import java.util.Map;
+
+public abstract class TaskDivideStrategy {
+
+ public abstract List<KeyValue> divide(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc);
+
+}
[rocketmq-connect] 10/43: Add SourceJdbcTask and Schema
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 9ea583f8516e23957b8650eb5eb9b97ae6857caf
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Mon Jul 29 22:12:27 2019 +0800
Add SourceJdbcTask and Schema
---
README.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 74 insertions(+)
diff --git a/README.md b/README.md
index a3ebb26..f02d838 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,75 @@
# RocketMQ-connect-jdbc
+
+### Directory Structure Description
+
+```web-idl
+│ pom.xml
+│ README.md
+└─src
+ ├─main
+ │ └─java
+ │ └─org
+ │ └─apache
+ │ └─rocketmq
+ │ └─connect
+ │ └─jdbc
+ │ │ Config.java
+ │ ├─connector
+ │ │ JdbcSourceConnector.java
+ │ │ JdbcSourceTask.java
+ │ ├─dialect
+ │ ├─schema
+ │ │ │ Database.java
+ │ │ │ Schema.java
+ │ │ │ Table.java
+ │ │ │
+ │ │ └─column
+ │ │ BigIntColumnParser.java
+ │ │ ColumnParser.java
+ │ │ DateTimeColumnParser.java
+ │ │ DefaultColumnParser.java
+ │ │ EnumColumnParser.java
+ │ │ IntColumnParser.java
+ │ │ SetColumnParser.java
+ │ │ StringColumnParser.java
+ │ │ TimeColumnParser.java
+ │ │ YearColumnParser.java
+ │ ├─sink
+ │ └─source
+ │ Querier.java
+ └─test
+ └─java
+ └─org
+ └─apache
+ └─rocketmq
+ └─connect
+ └─jdbc
+ └─connector
+ JdbcSourceConnectorTest.java
+ JdbcSourceTaskTest.java
+```
+
+### Some Result of Testing JdbcSourceTask
+
+
+
+#### Data Type：SourceDataEntry
+
+{sourcePartition,sourcePosition,DataEntry{timestamp,entryType=CREATE,queueName,shardingKey,schema.schema=Schema{dataSource=DATABASE_NAME,name=TABLE_NAME,fields=[Field{index,name,type}]},payloading}}
+
+- For example
+
+```javascript
+SourceDataEntry{sourcePartition=java.nio.HeapByteBuffer[pos=0 lim=14 cap=14], sourcePosition=java.nio.HeapByteBuffer[pos=0 lim=44 cap=44]} DataEntry{timestamp=1564397062419, entryType=CREATE, queueName='student', shardingKey='null',
+schema=Schema{dataSource='jdbc_db', name='student', fields=[Field{index=0, name='id', type=INT32}, Field{index=1, name='first', type=STRING},
+Field{index=2, name='last', type=STRING}, Field{index=3, name='age', type=INT32}]}, payload=[102121, "Python", "Py", 25]}
+```
+
+#### Mentioned DataBase Information and all SourceDataEntry
+
+- For example
+
+
+
+
+
[rocketmq-connect] 32/43: fix(jdbc-connect) removed unused class
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 7ebfedf7e367028c5bf413415794f72c08700956
Author: 翊名 <du...@alibaba-inc.com>
AuthorDate: Mon Mar 30 12:53:24 2020 +0800
fix(jdbc-connect) removed unused class
---
.../rocketmq/connect/jdbc/config/Config.java | 31 ++++++++
.../jdbc/strategy/DivideTaskByConsistentHash.java | 82 ----------------------
.../jdbc/connector/JdbcSourceConnectorTest.java | 4 +-
3 files changed, 32 insertions(+), 85 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index ae86d3f..7b4aca3 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
@@ -78,6 +78,9 @@ public class Config {
private long timestampDelayInterval = 0;
private String dbTimezone = "GMT+8";
private String queueName;
+ private String jdbcUrl;
+ private String jdbcUsername;
+ private String jdbcPassword;
private Logger log = LoggerFactory.getLogger(Config.class);
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
@@ -322,4 +325,32 @@ public class Config {
public void setWhiteTable(String whiteTable) {
this.whiteTable = whiteTable;
}
+
+ public void setPollInterval(long pollInterval) {
+ this.pollInterval = pollInterval;
+ }
+
+ public String getJdbcUrl() {
+ return jdbcUrl;
+ }
+
+ public void setJdbcUrl(String jdbcUrl) {
+ this.jdbcUrl = jdbcUrl;
+ }
+
+ public String getJdbcUsername() {
+ return jdbcUsername;
+ }
+
+ public void setJdbcUsername(String jdbcUsername) {
+ this.jdbcUsername = jdbcUsername;
+ }
+
+ public String getJdbcPassword() {
+ return jdbcPassword;
+ }
+
+ public void setJdbcPassword(String jdbcPassword) {
+ this.jdbcPassword = jdbcPassword;
+ }
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java
deleted file mode 100644
index bac7358..0000000
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java
+++ /dev/null
@@ -1,82 +0,0 @@
-package org.apache.rocketmq.connect.jdbc.strategy;/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import com.alibaba.fastjson.JSONObject;
-import io.openmessaging.KeyValue;
-import io.openmessaging.internal.DefaultKeyValue;
-import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter;
-import org.apache.rocketmq.common.consistenthash.Node;
-import org.apache.rocketmq.replicator.config.DataType;
-import org.apache.rocketmq.replicator.config.TaskConfigEnum;
-import org.apache.rocketmq.replicator.config.TaskDivideConfig;
-import org.apache.rocketmq.replicator.config.TaskTopicInfo;
-
-import java.util.*;
-
-public class DivideTaskByConsistentHash extends TaskDivideStrategy {
- @Override public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
-
- List<KeyValue> config = new ArrayList<>();
- int parallelism = tdc.getTaskParallelism();
- Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
- int id = -1;
-
- Collection<ClientNode> cidNodes = new ArrayList<>();
- for (int i = 0; i < parallelism; i++) {
- cidNodes.add(new ClientNode(i, Integer.toString(i)));
- queueTopicList.put(i, new ArrayList<>());
- }
-
- ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
-
- for (String t : topicMap.keySet()) {
- for (TaskTopicInfo queue : topicMap.get(t)) {
- ClientNode clientNode = router.routeNode(queue.toString());
- if (clientNode != null) {
- queueTopicList.get(clientNode.index).add(queue);
- }
- }
- }
-
- for (int i = 0; i < parallelism; i++) {
- KeyValue keyValue = new DefaultKeyValue();
- keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
- keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
- keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
- keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
- keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
- config.add(keyValue);
- }
-
- return config;
- }
-
- private static class ClientNode implements Node {
- private final String clientID;
- private final int index;
-
- public ClientNode(int index, String clientID) {
- this.index = index;
- this.clientID = clientID;
- }
-
- @Override
- public String getKey() {
- return clientID;
- }
- }
-}
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
index 1e7cf78..5d25f98 100644
--- a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
@@ -22,9 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.util.HashSet;
import java.util.Set;
-import org.apache.rocketmq.connect.jdbc.Config;
-import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
-
+import org.apache.rocketmq.connect.jdbc.config.Config;
import org.junit.Test;
import io.openmessaging.KeyValue;
[rocketmq-connect] 03/43: Update pom.xml
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 9716b10fa3fb9374e9a3ab181f4bfc27a3eeed86
Author: Yuchen Li <yu...@126.com>
AuthorDate: Fri Jul 19 14:19:37 2019 +0800
Update pom.xml
---
pom.xml | 6 ------
1 file changed, 6 deletions(-)
diff --git a/pom.xml b/pom.xml
index 361dd77..a53b8b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -184,12 +184,6 @@
<artifactId>commons-cli</artifactId>
<version>1.2</version>
</dependency>
- <dependency>
- <groupId>javax.jms</groupId>
- <artifactId>javax.jms-api</artifactId>
- <version>2.0</version>
- </dependency>
-
</dependencies>
</project>
[rocketmq-connect] 14/43: develop the jdbcsource connector
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 488223c3aa3d15ce4676e6442d894cb99b06275e
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Fri Aug 9 12:04:51 2019 +0800
develop the jdbcsource connector
---
README.md | 43 ++-
pom.xml | 8 +
.../org/apache/rocketmq/connect/jdbc/Config.java | 52 ++--
.../jdbc/connector/JdbcSourceConnector.java | 12 +-
.../connect/jdbc/connector/JdbcSourceTask.java | 39 +--
.../rocketmq/connect/jdbc/schema/Database.java | 4 +-
.../rocketmq/connect/jdbc/schema/Schema.java | 4 +-
.../apache/rocketmq/connect/jdbc/schema/Table.java | 36 +--
.../rocketmq/connect/jdbc/source/Querier.java | 314 ++++++++++-----------
9 files changed, 270 insertions(+), 242 deletions(-)
diff --git a/README.md b/README.md
index 96da884..32da4b5 100644
--- a/README.md
+++ b/README.md
@@ -149,34 +149,55 @@ httpPort=8081
看到日志目录查看connect_runtime.log
-如果看到以下日志说明runttiime启动成功了
-
-2019-07-16 10:56:24 INFO RebalanceService - RebalanceService service started
-2019-07-16 10:56:24 INFO main - The worker [DEFAULT_WORKER_1] boot success.
-
-2、启动sourceConnector
-
- 正在做测试（To be continued）已实现Bulk Mode
+windows用户可以用CMD到程序根目录下再输入：
+```
cd target/distribution/
java -cp .;./conf/;./lib/* org.apache.rocketmq.connect.runtime.ConnectStartup -c conf/connect.conf
+```
+如果看到以下日志说明runttiime启动成功了
+2019-07-16 10:56:24 INFO RebalanceService - RebalanceService service started
+2019-07-16 10:56:24 INFO main - The worker [DEFAULT_WORKER_1] boot success.
+
+2、启动sourceConnector
-在http中输入Get 请求
+```
+1、git clone https://github.com/apache/rocketmq-externals.git
+2、cd rocketmq-externals/rocketmq-connect-jdbc
+3、mvn -Dmaven.test.skip=true package
-示例
+```
-[http://127.0.0.1:8085/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"bulk","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}](http://127.0.0.1:8085/connectors/testSourceConnector1?config={% [...]
+- 复制第三方jar至target
+```
+mvn dependency:copy-dependencies
+```
+已实现Bulk查询方法，在http中输入Get 请求（目前仅适配过MYSQL）
+```http
+http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"bulk","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+```
+看到一下日志说明Jdbc source connector启动成功了
+2019-08-09 11:33:22 INFO RebalanceService - JdbcSourceConnector verifyAndSetConfig enter
+2019-08-09 11:33:23 INFO pool-9-thread-1 - Config.load.start
+2019-08-09 11:33:23 INFO pool-9-thread-1 - querier.start
+2019-08-09 11:33:23 INFO pool-9-thread-1 - {password=199812160, validationQuery=SELECT 1 FROM DUAL, testWhileIdle=true, timeBetweenEvictionRunsMillis=60000, minEvictableIdleTimeMillis=300000, initialSize=2, driverClassName=com.mysql.cj.jdbc.Driver, maxWait=60000, url=jdbc:mysql://localhost:3306?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8, username=root, maxActive=2},config read successful
+2019-08-09 11:33:24 INFO RebalanceService - JdbcSourceConnector verifyAndSetConfig enter
+2019-08-09 11:33:25 INFO pool-9-thread-1 - {dataSource-1} inited
+2019-08-09 11:33:27 INFO pool-9-thread-1 - schema load successful
+2019-08-09 11:33:27 INFO pool-9-thread-1 - querier.poll
+3、启动sinkConnector
+To Be Continued.
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 830f9ed..1d708f3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -54,6 +54,14 @@
<artifactId>clirr-maven-plugin</artifactId>
<version>2.7</version>
</plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <configuration>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ <excludeTransitive>false</excludeTransitive>
+ <stripVersion>true</stripVersion>
+ </configuration>
+ </plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.6.1</version>
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
index 4f7456b..533d53b 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
@@ -25,56 +25,56 @@ import java.lang.reflect.Method;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+
public class Config {
@SuppressWarnings("serial")
private static final Logger LOG = LoggerFactory.getLogger(Config.class);
/* Database Connection Config */
- public String jdbcUrl="localhost:3306";
- public String jdbcUsername="root";
- public String jdbcPassword="199812160";
+ public String jdbcUrl = "localhost:3306";
+ public String jdbcUsername = "root";
+ public String jdbcPassword = "199812160";
public String rocketmqTopic;
public String jdbcBackoff;
public String jdbcAttempts;
- public String catalogPattern=null;
+ public String catalogPattern = null;
public List tableWhitelist;
public List tableBlacklist;
- public String schemaPattern=null;
- public boolean numericPrecisionMapping=false;
- public String bumericMapping=null;
- public String dialectName="";
+ public String schemaPattern = null;
+ public boolean numericPrecisionMapping = false;
+ public String bumericMapping = null;
+ public String dialectName = "";
/* Mode Config */
- public String mode="";
- public String incrementingColumnName= "";
- public String query="";
- public String timestampColmnName="";
- public boolean validateNonNull=true;
+ public String mode = "";
+ public String incrementingColumnName = "";
+ public String query = "";
+ public String timestampColmnName = "";
+ public boolean validateNonNull = true;
/*Connector config*/
- public String tableTypes="table";
- public long pollInterval=5000;
- public int batchMaxRows=100;
- public long tablePollInterval=60000;
- public long timestampDelayInterval=0;
- public String dbTimezone="UTC";
+ public String tableTypes = "table";
+ public long pollInterval = 5000;
+ public int batchMaxRows = 100;
+ public long tablePollInterval = 60000;
+ public long timestampDelayInterval = 0;
+ public String dbTimezone = "UTC";
public String queueName;
private Logger log = LoggerFactory.getLogger(Config.class);
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
{
- // add("jdbcUrl");
- // add("jdbcUsername");
- // add("jdbcPassword");
- // add("mode");
- // add("rocketmqTopic");
+ add("jdbcUrl");
+ add("jdbcUsername");
+ add("jdbcPassword");
+ // add("mode");
+ // add("rocketmqTopic");
}
};
-
public void load(KeyValue props) {
- log.info("Config.load.start");
+ log.info("Config.load.start");
properties2Object(props, this);
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
index 8c30a62..bdbeb8b 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
@@ -37,7 +37,7 @@ public class JdbcSourceConnector extends SourceConnector {
@Override
public String verifyAndSetConfig(KeyValue config) {
- log.info("1216123 JdbcSourceConnector verifyAndSetConfig enter");
+ log.info("JdbcSourceConnector verifyAndSetConfig enter");
for (String requestKey : Config.REQUEST_CONFIG) {
if (!config.containsKey(requestKey)) {
@@ -45,7 +45,7 @@ public class JdbcSourceConnector extends SourceConnector {
}
}
this.config = config;
-
+
return "";
}
@@ -68,13 +68,13 @@ public class JdbcSourceConnector extends SourceConnector {
}
@Override
- public Class<? extends Task> taskClass(){
- return JdbcSourceTask.class;
- }
+ public Class<? extends Task> taskClass() {
+ return JdbcSourceTask.class;
+ }
@Override
public List<KeyValue> taskConfigs() {
- log.info("List.start");
+ log.info("List.start");
List<KeyValue> config = new ArrayList<>();
config.add(this.config);
return config;
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index 78f1809..91659ec 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -18,6 +18,7 @@
*/
package org.apache.rocketmq.connect.jdbc.connector;
+
import io.openmessaging.connector.api.source.SourceTask;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -47,32 +48,32 @@ public class JdbcSourceTask extends SourceTask {
private static final Logger log = LoggerFactory.getLogger(JdbcSourceTask.class);
private Config config;
-
- private List<Table> list=new LinkedList<>();
-
- Querier querier = new Querier();
+
+ private List<Table> list = new LinkedList<>();
+
+ Querier querier = new Querier();
+
@Override
public Collection<SourceDataEntry> poll() {
List<SourceDataEntry> res = new ArrayList<>();
try {
-
+
JSONObject jsonObject = new JSONObject();
jsonObject.put("nextQuery", "database");
jsonObject.put("nextPosition", "10");
- //To be Continued
- log.info("querier.poll");
+ //To be Continued
querier.poll();
- log.info("1216connector.start");
- int mm=0;
- for(Table dataRow : querier.getList()){
- System.out.println(dataRow.getColList().get(0));
- log.info("xunhuankaishi");
+ log.info("querier.poll, start");
+ int mm = 0;
+ for (Table dataRow : querier.getList()) {
+ System.out.println(dataRow.getColList().get(0));
+ log.info("xunhuankaishi");
log.info("Received {} record: {} ", dataRow.getColList().get(0), mm++);
- Schema schema = new Schema();
+ Schema schema = new Schema();
schema.setDataSource(dataRow.getDatabase());
schema.setName(dataRow.getName());
schema.setFields(new ArrayList<>());
- for(int i = 0; i < dataRow.getColList().size(); i++){
+ for (int i = 0; i < dataRow.getColList().size(); i++) {
String columnName = dataRow.getColList().get(i);
String rawDataType = dataRow.getRawDataTypeList().get(i);
Field field = new Field(i, columnName, ColumnParser.mapConnectorFieldType(rawDataType));
@@ -82,10 +83,10 @@ public class JdbcSourceTask extends SourceTask {
dataEntryBuilder.timestamp(System.currentTimeMillis())
.queue(dataRow.getName())
.entryType(EntryType.CREATE);
- for(int i = 0; i < dataRow.getColList().size(); i++){
- Object value=dataRow.getDataList().get(i);
+ for (int i = 0; i < dataRow.getColList().size(); i++) {
+ Object value = dataRow.getDataList().get(i);
System.out.println(1);
- System.out.println(dataRow.getColList().get(i)+"|"+value);
+ System.out.println(dataRow.getColList().get(i) + "|" + value);
dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSON.toJSONString(value));
}
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
@@ -104,8 +105,8 @@ public class JdbcSourceTask extends SourceTask {
try {
this.config = new Config();
this.config.load(props);
- log.info("querier.start");
- querier.start();
+ log.info("querier.start");
+ querier.start();
} catch (Exception e) {
log.error("JDBC task start failed.", e);
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
index b88661d..657ebb8 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
@@ -18,6 +18,7 @@
package org.apache.rocketmq.connect.jdbc.schema;
//import io.openmessaging.mysql.binlog.EventProcessor;
+
import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -36,6 +37,7 @@ public class Database {
private String name;
private DataSource dataSource;
public Map<String, Table> tableMap = new HashMap<String, Table>();
+
public Database(String name, DataSource dataSource) {
this.name = name;
this.dataSource = dataSource;
@@ -59,7 +61,7 @@ public class Database {
String dataType = rs.getString(3);
String colType = rs.getString(4);
String charset = rs.getString(5);
-
+
ColumnParser columnParser = ColumnParser.getColumnParser(dataType, colType, charset);
if (!tableMap.containsKey(tableName)) {
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
index 6ce6621..7434bbc 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
@@ -116,8 +116,8 @@ public class Schema {
load();
break;
} catch (Exception e) {
- // LOGGER.error("Reload schema error.", e);
- System.out.println("Reload schema error."+e);
+ // LOGGER.error("Reload schema error.", e);
+ System.out.println("Reload schema error." + e);
}
}
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
index c0d793d..8c9a42d 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
@@ -28,7 +28,7 @@ public class Table {
private List<String> colList = new LinkedList<>();
private List<ColumnParser> parserList = new LinkedList<>();
private List<String> rawDataTypeList = new LinkedList<>();
- private List<Object> dataList =new LinkedList<>();
+ private List<Object> dataList = new LinkedList<>();
public Table(String database, String table) {
this.database = database;
@@ -40,18 +40,18 @@ public class Table {
}
public void setParserList(List<ColumnParser> parserList) {
- this.parserList = parserList;
- }
+ this.parserList = parserList;
+ }
- public void setRawDataTypeList(List<String> rawDataTypeList) {
- this.rawDataTypeList = rawDataTypeList;
- }
+ public void setRawDataTypeList(List<String> rawDataTypeList) {
+ this.rawDataTypeList = rawDataTypeList;
+ }
- public void addParser(ColumnParser columnParser) {
+ public void addParser(ColumnParser columnParser) {
parserList.add(columnParser);
}
- public void addRawDataType(String rawDataType){
+ public void addRawDataType(String rawDataType) {
this.rawDataTypeList.add(rawDataType);
}
@@ -75,16 +75,16 @@ public class Table {
return parserList;
}
- public List<Object> getDataList() {
- return dataList;
- }
+ public List<Object> getDataList() {
+ return dataList;
+ }
+
+ public void setDataList(List<Object> dataList) {
+ this.dataList = dataList;
+ }
- public void setDataList(List<Object> dataList) {
- this.dataList = dataList;
- }
+ public void setColList(List<String> colList) {
+ this.colList = colList;
+ }
- public void setColList(List<String> colList) {
- this.colList = colList;
- }
-
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index 1f630ea..3d5b2c6 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -22,168 +22,164 @@ import org.slf4j.LoggerFactory;
import com.alibaba.druid.pool.DruidDataSourceFactory;
-
import org.apache.rocketmq.connect.jdbc.schema.*;
import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
public class Querier {
- static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
- private final Logger log = LoggerFactory.getLogger(getClass()); // use concrete subclass
- protected String topicPrefix;
- protected String jdbcUrl;
- private final Queue<Connection> connections = new ConcurrentLinkedQueue<>();
- private Config config = new Config();
- private DataSource dataSource;
- private List<Table> list=new LinkedList<>();
-
-
-
- public List<Table> getList() {
- return list;
- }
-
- public void setList(List<Table> list) {
- this.list = list;
- }
-
- public Connection getConnection() throws SQLException {
-
- // These config names are the same for both source and sink configs ...
- String username = config.jdbcUsername;
- String dbPassword = config.jdbcPassword;
- jdbcUrl = config.jdbcUrl;
- Properties properties = new Properties();
- if (username != null) {
- properties.setProperty("user", username);
- }
- if (dbPassword != null) {
- properties.setProperty("password", dbPassword);
- }
- Connection connection = DriverManager.getConnection(jdbcUrl, properties);
-
- connections.add(connection);
- return connection;
- }
-
- public void stop() {
- Connection conn;
- while ((conn = connections.poll()) != null) {
- try {
- conn.close();
- } catch (Throwable e) {
- log.warn("Error while closing connection to {}", "jdbc", e);
- }
- }
- }
-
- protected PreparedStatement createDBPreparedStatement(Connection db) throws SQLException {
-
- String SQL = "select table_name,column_name,data_type,column_type,character_set_name "
- + "from information_schema.columns " + "where table_schema = jdbc_db order by ORDINAL_POSITION";
-
- log.trace("Creating a PreparedStatement '{}'", SQL);
- PreparedStatement stmt = db.prepareStatement(SQL);
- return stmt;
-
- }
-
- protected PreparedStatement createPreparedStatement(Connection db, String string) throws SQLException {
- String query = "select * from " + string;
- log.trace("Creating a PreparedStatement '{}'", query);
- PreparedStatement stmt = db.prepareStatement(query);
- return stmt;
-
- }
-
- protected ResultSet executeQuery(PreparedStatement stmt) throws SQLException {
- return stmt.executeQuery();
- }
-
- public static void main(String[] args) throws Exception {
- Querier querier = new Querier();
- try {
- querier.start();
- querier.poll();
-
- } catch (SQLException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- }
-
- private Schema schema;
-
-
- public void poll() {
- try {
-
- PreparedStatement stmt;
- String query = "select * from ";
- Connection conn = dataSource.getConnection();
-
- for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
- String db = entry.getKey();
- if(!db.contains("jdbc_db"))
- continue;
- Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
- while (iterator.hasNext()) {
- Map.Entry<String, Table> tableEntry = iterator.next();
- String tb=tableEntry.getKey();
- stmt = conn.prepareStatement(query+db + "." +tb);
- ResultSet rs;
- rs = stmt.executeQuery();
- List<String> colList = tableEntry.getValue().getColList();
- List<String> DataTypeList = tableEntry.getValue().getRawDataTypeList();
- List<ColumnParser> ParserList = tableEntry.getValue().getParserList();
-
- while(rs.next()) {
- Table table=new Table(db, tb);
- System.out.print("|");
- table.setColList(colList);
- table.setRawDataTypeList(DataTypeList);
- table.setParserList(ParserList);
-
- for (String string : colList) {
- table.getDataList().add(rs.getObject(string));
- System.out.print(string+" : "+rs.getObject(string)+"|");
- }
- list.add(table);
- System.out.println();
- }
- }
- }
-
- } catch (SQLException e) {
- e.printStackTrace();
- }
-
- }
-
- public void start() throws Exception {
- initDataSource();
- schema = new Schema(dataSource);
- schema.load();
- log.info("schema load successful");
- }
-
- private void initDataSource() throws Exception {
- Map<String, String> map = new HashMap<>();
- map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
- map.put("url",
- "jdbc:mysql://" + config.jdbcUrl + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
- map.put("username", config.jdbcUsername);
- map.put("password", config.jdbcPassword);
- map.put("initialSize", "2");
- map.put("maxActive", "2");
- map.put("maxWait", "60000");
- map.put("timeBetweenEvictionRunsMillis", "60000");
- map.put("minEvictableIdleTimeMillis", "300000");
- map.put("validationQuery", "SELECT 1 FROM DUAL");
- map.put("testWhileIdle", "true");
- log.info("{},config read successful",map);
- dataSource = DruidDataSourceFactory.createDataSource(map);
-
- }
+ static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver";
+ private final Logger log = LoggerFactory.getLogger(getClass()); // use concrete subclass
+ protected String topicPrefix;
+ protected String jdbcUrl;
+ private final Queue<Connection> connections = new ConcurrentLinkedQueue<>();
+ private Config config = new Config();
+ private DataSource dataSource;
+ private List<Table> list = new LinkedList<>();
+
+ public List<Table> getList() {
+ return list;
+ }
+
+ public void setList(List<Table> list) {
+ this.list = list;
+ }
+
+ public Connection getConnection() throws SQLException {
+
+ // These config names are the same for both source and sink configs ...
+ String username = config.jdbcUsername;
+ String dbPassword = config.jdbcPassword;
+ jdbcUrl = config.jdbcUrl;
+ Properties properties = new Properties();
+ if (username != null) {
+ properties.setProperty("user", username);
+ }
+ if (dbPassword != null) {
+ properties.setProperty("password", dbPassword);
+ }
+ Connection connection = DriverManager.getConnection(jdbcUrl, properties);
+
+ connections.add(connection);
+ return connection;
+ }
+
+ public void stop() {
+ Connection conn;
+ while ((conn = connections.poll()) != null) {
+ try {
+ conn.close();
+ } catch (Throwable e) {
+ log.warn("Error while closing connection to {}", "jdbc", e);
+ }
+ }
+ }
+
+ protected PreparedStatement createDBPreparedStatement(Connection db) throws SQLException {
+
+ String SQL = "select table_name,column_name,data_type,column_type,character_set_name "
+ + "from information_schema.columns " + "where table_schema = jdbc_db order by ORDINAL_POSITION";
+
+ log.trace("Creating a PreparedStatement '{}'", SQL);
+ PreparedStatement stmt = db.prepareStatement(SQL);
+ return stmt;
+
+ }
+
+ protected PreparedStatement createPreparedStatement(Connection db, String string) throws SQLException {
+ String query = "select * from " + string;
+ log.trace("Creating a PreparedStatement '{}'", query);
+ PreparedStatement stmt = db.prepareStatement(query);
+ return stmt;
+
+ }
+
+ protected ResultSet executeQuery(PreparedStatement stmt) throws SQLException {
+ return stmt.executeQuery();
+ }
+
+ public static void main(String[] args) throws Exception {
+ Querier querier = new Querier();
+ try {
+ querier.start();
+ querier.poll();
+
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ }
+
+ private Schema schema;
+
+ public void poll() {
+ try {
+
+ PreparedStatement stmt;
+ String query = "select * from ";
+ Connection conn = dataSource.getConnection();
+
+ for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
+ String db = entry.getKey();
+ if (!db.contains("jdbc_db"))
+ continue;
+ Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<String, Table> tableEntry = iterator.next();
+ String tb = tableEntry.getKey();
+ stmt = conn.prepareStatement(query + db + "." + tb);
+ ResultSet rs;
+ rs = stmt.executeQuery();
+ List<String> colList = tableEntry.getValue().getColList();
+ List<String> DataTypeList = tableEntry.getValue().getRawDataTypeList();
+ List<ColumnParser> ParserList = tableEntry.getValue().getParserList();
+
+ while (rs.next()) {
+ Table table = new Table(db, tb);
+ System.out.print("|");
+ table.setColList(colList);
+ table.setRawDataTypeList(DataTypeList);
+ table.setParserList(ParserList);
+
+ for (String string : colList) {
+ table.getDataList().add(rs.getObject(string));
+ System.out.print(string + " : " + rs.getObject(string) + "|");
+ }
+ list.add(table);
+ System.out.println();
+ }
+ }
+ }
+
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ public void start() throws Exception {
+ initDataSource();
+ schema = new Schema(dataSource);
+ schema.load();
+ log.info("schema load successful");
+ }
+
+ private void initDataSource() throws Exception {
+ Map<String, String> map = new HashMap<>();
+ map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
+ map.put("url",
+ "jdbc:mysql://" + config.jdbcUrl + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
+ map.put("username", config.jdbcUsername);
+ map.put("password", config.jdbcPassword);
+ map.put("initialSize", "2");
+ map.put("maxActive", "2");
+ map.put("maxWait", "60000");
+ map.put("timeBetweenEvictionRunsMillis", "60000");
+ map.put("minEvictableIdleTimeMillis", "300000");
+ map.put("validationQuery", "SELECT 1 FROM DUAL");
+ map.put("testWhileIdle", "true");
+ log.info("{},config read successful", map);
+ dataSource = DruidDataSourceFactory.createDataSource(map);
+
+ }
}
[rocketmq-connect] 34/43: 1.add rocketmq-tools dependency so rocketmq-connect-jdbc can run on it (#537)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit e325d72a4b1cb13bbb101082301d2adb530a2c00
Author: affe <af...@gmail.com>
AuthorDate: Sun Mar 29 22:07:17 2020 -0700
1.add rocketmq-tools dependency so rocketmq-connect-jdbc can run on it (#537)
---
pom.xml | 1 -
1 file changed, 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 6beb19c..9df23c4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -264,7 +264,6 @@
<artifactId>druid</artifactId>
<version>1.0.31</version>
</dependency>
-
</dependencies>
</project>
[rocketmq-connect] 11/43: Add JdbcSourceTask and Schema
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 8ecde453ec4680fd8b37538bd7bd6312cce7fc00
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Fri Aug 2 00:20:44 2019 +0800
Add JdbcSourceTask and Schema
---
README.md | 113 ++++++++++++++++++++-
lib/mysql-connector-java-8.0.11.jar | Bin 0 -> 2036609 bytes
pom.xml | 39 ++++++-
.../org/apache/rocketmq/connect/jdbc/Config.java | 13 +--
.../jdbc/connector/JdbcSourceConnector.java | 9 +-
.../connect/jdbc/connector/JdbcSourceTask.java | 16 +--
.../rocketmq/connect/jdbc/schema/Database.java | 2 -
.../rocketmq/connect/jdbc/source/Querier.java | 8 +-
.../rocketmq/connect/jdbc/ReplicatorTest.java | 74 ++++++++++++++
.../jdbc/connector/JdbcSourceConnectorTest.java | 6 +-
10 files changed, 254 insertions(+), 26 deletions(-)
diff --git a/README.md b/README.md
index f02d838..96da884 100644
--- a/README.md
+++ b/README.md
@@ -60,9 +60,9 @@
- For example
```javascript
-SourceDataEntry{sourcePartition=java.nio.HeapByteBuffer[pos=0 lim=14 cap=14], sourcePosition=java.nio.HeapByteBuffer[pos=0 lim=44 cap=44]} DataEntry{timestamp=1564397062419, entryType=CREATE, queueName='student', shardingKey='null',
-schema=Schema{dataSource='jdbc_db', name='student', fields=[Field{index=0, name='id', type=INT32}, Field{index=1, name='first', type=STRING},
-Field{index=2, name='last', type=STRING}, Field{index=3, name='age', type=INT32}]}, payload=[102121, "Python", "Py", 25]}
+ SourceDataEntry{sourcePartition=java.nio.HeapByteBuffer[pos=0 lim=14 cap=14], sourcePosition=java.nio.HeapByteBuffer[pos=0 lim=44 cap=44]} DataEntry{timestamp=1564397062419, entryType=CREATE, queueName='student', shardingKey='null',
+ schema=Schema{dataSource='jdbc_db', name='student', fields=[Field{index=0, name='id', type=INT32}, Field{index=1, name='first', type=STRING},
+ Field{index=2, name='last', type=STRING}, Field{index=3, name='age', type=INT32}]}, payload=[102121, "Python", "Py", 25]}
```
#### Mentioned DataBase Information and all SourceDataEntry
@@ -73,3 +73,110 @@ Field{index=2, name='last', type=STRING}, Field{index=3, name='age', type=INT32}
+**启动Connector**
+
+[http://127.0.0.1:8081/connectors/connector-name?config={"connector-class":"org.apache.rocketmq.connect.kafka.connector.KafkaSourceConnector","oms-driver-url":"oms](http://127.0.0.1:8081/connectors/connector-name?config=%7B%22connector-class%22:%22org.apache.rocketmq.connect.kafka.connector.KafkaSourceConnector%22,%22oms-driver-url%22:%22oms): rocketmq://127.0.0.1:9876/default:default","tasks.num":"1","kafka.topics":"test1,test2","kafka.group.id":"group0","kafka.bootstrap.server":"127.0. [...]
+
+**查看Connector运行状态**
+
+<http://127.0.0.1:8081/connectors/connector-name/status>
+
+**查看Connector配置**
+
+<http://127.0.0.1:8081/connectors/connector-name/config>
+
+**关闭Connector**
+
+<http://127.0.0.1:8081/connectors/connector-name/stop>
+
+
+
+
+
+
+
+# JDBC Connector 构建
+
+
+
+#### 一、下载rocketmq-connect-runtime
+
+```
+1、git clone https://github.com/apache/rocketmq-externals.git
+
+2、cd rocketmq-externals/rocketmq-connect-runtime
+
+3、mvn -Dmaven.test.skip=true package
+
+4、cd target/distribution/conf
+```
+
+- a、修改connect.conf配置文件
+
+```
+#1、rocketmq 配置
+namesrvAddr=127.0.0.1:9876
+
+#2、file-connect jar包路径
+pluginPaths=/home/connect/file-connect/target
+
+#3、runtime持久化文件目录
+storePathRootDir=/home/connect/storeRoot
+
+#4、http服务端口
+httpPort=8081
+```
+
+
+
+
+
+- b、日志相关配置在logback.xml中修改
+
+```
+注：rocketmq需要先创建cluster-topic，config-topic，offset-topic，position-topic
+4个topic，并且为了保证消息有序，每个topic可以只一个queue
+```
+
+### 二、启动Connector
+
+1、启动runtime
+回到rocketmq-externals/rocketmq-connect-runtime目录
+
+```
+./run_worker.sh
+```
+
+看到日志目录查看connect_runtime.log
+
+如果看到以下日志说明runttiime启动成功了
+
+2019-07-16 10:56:24 INFO RebalanceService - RebalanceService service started
+2019-07-16 10:56:24 INFO main - The worker [DEFAULT_WORKER_1] boot success.
+
+2、启动sourceConnector
+
+ 正在做测试（To be continued）已实现Bulk Mode
+
+cd target/distribution/
+
+java -cp .;./conf/;./lib/* org.apache.rocketmq.connect.runtime.ConnectStartup -c conf/connect.conf
+
+
+
+在http中输入Get 请求
+
+
+
+示例
+
+[http://127.0.0.1:8085/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"bulk","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}](http://127.0.0.1:8085/connectors/testSourceConnector1?config={% [...]
+
+
+
+
+
+
+
+
+
diff --git a/lib/mysql-connector-java-8.0.11.jar b/lib/mysql-connector-java-8.0.11.jar
new file mode 100644
index 0000000..c8b8b3f
Binary files /dev/null and b/lib/mysql-connector-java-8.0.11.jar differ
diff --git a/pom.xml b/pom.xml
index dee7710..830f9ed 100644
--- a/pom.xml
+++ b/pom.xml
@@ -127,6 +127,29 @@
<artifactId>findbugs-maven-plugin</artifactId>
<version>3.0.4</version>
</plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>3.0.0</version>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.rocketmq.connect.jdbc.jdbcSourceConnector</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
@@ -159,6 +182,11 @@
<artifactId>openmessaging-connector</artifactId>
<version>0.1.0-beta</version>
</dependency>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-api</artifactId>
+ <version>0.3.1-alpha</version>
+ </dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
@@ -194,7 +222,16 @@
<artifactId>commons-cli</artifactId>
<version>1.2</version>
</dependency>
-
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>8.0.11</version>
+ </dependency>
+ <dependency>
+ <groupId>io.javalin</groupId>
+ <artifactId>javalin</artifactId>
+ <version>1.3.0</version>
+ </dependency>
</dependencies>
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
index 69ff9b0..4f7456b 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
@@ -61,19 +61,20 @@ public class Config {
public String dbTimezone="UTC";
public String queueName;
+ private Logger log = LoggerFactory.getLogger(Config.class);
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
{
- add("jdbcUrl");
- add("jdbcUsername");
- add("jdbcPassword");
- add("mode");
- add("rocketmqTopic");
+ // add("jdbcUrl");
+ // add("jdbcUsername");
+ // add("jdbcPassword");
+ // add("mode");
+ // add("rocketmqTopic");
}
};
public void load(KeyValue props) {
-
+ log.info("Config.load.start");
properties2Object(props, this);
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
index 8a6047c..8c30a62 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
@@ -26,7 +26,7 @@ import io.openmessaging.connector.api.Task;
import io.openmessaging.connector.api.source.SourceConnector;
import org.apache.rocketmq.connect.jdbc.Config;
-//import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
+import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,13 +36,16 @@ public class JdbcSourceConnector extends SourceConnector {
@Override
public String verifyAndSetConfig(KeyValue config) {
- log.info("JdbcSourceConnector verifyAndSetConfig enter");
+
+ log.info("1216123 JdbcSourceConnector verifyAndSetConfig enter");
for (String requestKey : Config.REQUEST_CONFIG) {
+
if (!config.containsKey(requestKey)) {
return "Request config key: " + requestKey;
}
}
this.config = config;
+
return "";
}
@@ -71,9 +74,9 @@ public class JdbcSourceConnector extends SourceConnector {
@Override
public List<KeyValue> taskConfigs() {
+ log.info("List.start");
List<KeyValue> config = new ArrayList<>();
config.add(this.config);
return config;
}
-
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index 32ea763..78f1809 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -25,7 +25,6 @@ import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import org.apache.rocketmq.connect.jdbc.Config;
-import org.apache.rocketmq.connect.jdbc.Replicator;
import org.apache.rocketmq.connect.jdbc.schema.Table;
import org.apache.rocketmq.connect.jdbc.source.Querier;
import org.apache.rocketmq.connect.jdbc.schema.column.*;
@@ -47,8 +46,6 @@ public class JdbcSourceTask extends SourceTask {
private static final Logger log = LoggerFactory.getLogger(JdbcSourceTask.class);
- private Replicator replicator;
-
private Config config;
private List<Table> list=new LinkedList<>();
@@ -58,15 +55,20 @@ public class JdbcSourceTask extends SourceTask {
public Collection<SourceDataEntry> poll() {
List<SourceDataEntry> res = new ArrayList<>();
try {
+
JSONObject jsonObject = new JSONObject();
jsonObject.put("nextQuery", "database");
jsonObject.put("nextPosition", "10");
//To be Continued
+ log.info("querier.poll");
querier.poll();
- System.out.println(querier.getList().size());
+ log.info("1216connector.start");
+ int mm=0;
for(Table dataRow : querier.getList()){
System.out.println(dataRow.getColList().get(0));
- Schema schema = new Schema();
+ log.info("xunhuankaishi");
+ log.info("Received {} record: {} ", dataRow.getColList().get(0), mm++);
+ Schema schema = new Schema();
schema.setDataSource(dataRow.getDatabase());
schema.setName(dataRow.getName());
schema.setFields(new ArrayList<>());
@@ -102,7 +104,9 @@ public class JdbcSourceTask extends SourceTask {
try {
this.config = new Config();
this.config.load(props);
+ log.info("querier.start");
querier.start();
+
} catch (Exception e) {
log.error("JDBC task start failed.", e);
}
@@ -110,7 +114,7 @@ public class JdbcSourceTask extends SourceTask {
@Override
public void stop() {
- replicator.stop();
+ querier.stop();
}
@Override public void pause() {
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
index 15fb77b..b88661d 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
@@ -87,8 +87,6 @@ public class Database {
private void addTable(String tableName) {
- // LOGGER.info("Schema load -- DATABASE:{},\tTABLE:{}", name, tableName);
-
Table table = new Table(name, tableName);
tableMap.put(tableName, table);
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index e99da74..1f630ea 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -65,7 +65,7 @@ public class Querier {
return connection;
}
- public void close() {
+ public void stop() {
Connection conn;
while ((conn = connections.poll()) != null) {
try {
@@ -114,7 +114,6 @@ public class Querier {
private Schema schema;
- private Map<Long, Table> tableMap = new HashMap<>();
public void poll() {
try {
@@ -125,6 +124,8 @@ public class Querier {
for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
String db = entry.getKey();
+ if(!db.contains("jdbc_db"))
+ continue;
Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
@@ -163,6 +164,7 @@ public class Querier {
initDataSource();
schema = new Schema(dataSource);
schema.load();
+ log.info("schema load successful");
}
private void initDataSource() throws Exception {
@@ -179,7 +181,9 @@ public class Querier {
map.put("minEvictableIdleTimeMillis", "300000");
map.put("validationQuery", "SELECT 1 FROM DUAL");
map.put("testWhileIdle", "true");
+ log.info("{},config read successful",map);
dataSource = DruidDataSourceFactory.createDataSource(map);
+
}
}
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/ReplicatorTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/ReplicatorTest.java
new file mode 100644
index 0000000..88d5586
--- /dev/null
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/ReplicatorTest.java
@@ -0,0 +1,74 @@
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with
+// * this work for additional information regarding copyright ownership.
+// * The ASF licenses this file to You under the Apache License, Version 2.0
+// * (the "License"); you may not use this file except in compliance with
+// * the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.rocketmq.connect.jms;
+//
+//import java.lang.reflect.Field;
+//
+//import javax.jms.Message;
+//
+//import org.apache.activemq.command.ActiveMQTextMessage;
+//import org.apache.rocketmq.connect.jms.pattern.PatternProcessor;
+//import org.junit.Before;
+//import org.junit.Test;
+//import org.mockito.Mockito;
+//
+//import org.junit.Assert;
+//
+//public class ReplicatorTest {
+//
+// Replicator replicator;
+//
+// PatternProcessor patternProcessor;
+//
+// Config config;
+//
+// @Before
+// public void before() throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException, SecurityException {
+// config = new Config();
+// replicator = new Replicator(config,null);
+//
+// patternProcessor = Mockito.mock(PatternProcessor.class);
+//
+// Field processor = Replicator.class.getDeclaredField("processor");
+// processor.setAccessible(true);
+// processor.set(replicator, patternProcessor);
+// }
+//
+// @Test(expected = RuntimeException.class)
+// public void startTest() throws Exception {
+// replicator.start();
+// }
+//
+// @Test
+// public void stop() throws Exception {
+// replicator.stop();
+// Mockito.verify(patternProcessor, Mockito.times(1)).stop();
+// }
+//
+// @Test
+// public void commitAddGetQueueTest() {
+// Message message = new ActiveMQTextMessage();
+// replicator.commit(message, false);
+// Assert.assertEquals(replicator.getQueue().poll(), message);
+// }
+//
+// @Test
+// public void getConfigTest() {
+// Assert.assertEquals(replicator.getConfig(), config);
+// }
+//}
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
index 97d87ee..297d517 100644
--- a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
@@ -38,8 +38,6 @@ public class JdbcSourceConnectorTest {
add("jdbcUrl");
add("jdbcUsername");
add("jdbcPassword");
- add("mode");
- add("rocketmqTopic");
}
};
@@ -52,7 +50,9 @@ public class JdbcSourceConnectorTest {
}
-
+// Set<String> getRequiredConfig() {
+// return REQUEST_CONFIG;
+// }
};
@Test
[rocketmq-connect] 39/43: [ISSUE #558] An ugly solution for fetch topic list error
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 1a49e60c049417dbc24df8e7f6b9fb5ac7ba0b71
Author: affe <af...@gmail.com>
AuthorDate: Mon Jul 27 20:28:21 2020 +0800
[ISSUE #558] An ugly solution for fetch topic list error
---
.../apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
index 6a41646..53379ec 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
@@ -11,6 +11,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.rocketmq.client.exception.MQClientException;
@@ -43,6 +44,7 @@ public class JdbcSinkConnector extends SinkConnector {
private volatile boolean adminStarted;
+ private ScheduledFuture<?> listenerHandle;
public JdbcSinkConnector() {
topicRouteMap = new HashMap<>();
dbConnectorConfig = new SinkDbConnectorConfig();
@@ -94,7 +96,7 @@ public class JdbcSinkConnector extends SinkConnector {
}
public void startListener() {
- executor.scheduleAtFixedRate(new Runnable() {
+ listenerHandle = executor.scheduleAtFixedRate(new Runnable() {
boolean first = true;
HashMap<String, Set<TaskTopicInfo>> origin = null;
@@ -169,9 +171,10 @@ public class JdbcSinkConnector extends SinkConnector {
}
}
+
@Override
public void stop() {
-
+ listenerHandle.cancel(true);
}
@Override
[rocketmq-connect] 19/43: Update Querier.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit e72688b6be2103d02ac7fe9313bdd786dc0c7c5f
Author: Yuchen Li <yu...@126.com>
AuthorDate: Fri Aug 16 14:20:54 2019 +0800
Update Querier.java
---
src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index 0907d40..13cbda5 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -144,8 +144,6 @@ public class Querier {
Connection conn = dataSource.getConnection();
for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
String db = entry.getKey();
- if (!db.contains("jdbc_db"))
- continue;
Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
[rocketmq-connect] 16/43: Update JdbcSourceConnectorTest.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit cd6410d0ce1c349c5215479e6a5db5e9e1d08f1c
Author: Yuchen Li <yu...@126.com>
AuthorDate: Mon Aug 12 17:00:36 2019 +0800
Update JdbcSourceConnectorTest.java
---
.../rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java | 3 ---
1 file changed, 3 deletions(-)
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
index 297d517..1e7cf78 100644
--- a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
@@ -50,9 +50,6 @@ public class JdbcSourceConnectorTest {
}
-// Set<String> getRequiredConfig() {
-// return REQUEST_CONFIG;
-// }
};
@Test
[rocketmq-connect] 13/43: delete lib
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit b1acd43c0b0aa5a1653b91296e7993d5c933f73a
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Wed Aug 7 14:03:11 2019 +0800
delete lib
---
lib/mysql-connector-java-8.0.11.jar | Bin 2036609 -> 0 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
diff --git a/lib/mysql-connector-java-8.0.11.jar b/lib/mysql-connector-java-8.0.11.jar
deleted file mode 100644
index c8b8b3f..0000000
Binary files a/lib/mysql-connector-java-8.0.11.jar and /dev/null differ
[rocketmq-connect] 25/43: [ISSUE #441] Add Jdbc Sink Connector (#442)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 182fd469bc8e9dc89bb62a23be66549a33e93967
Author: Eason Chen <qq...@gmail.com>
AuthorDate: Tue Nov 19 23:22:01 2019 +0800
[ISSUE #441] Add Jdbc Sink Connector (#442)
---
.../rocketmq/connect/jdbc/common/DBUtils.java | 211 +++++++++++++++++++++
.../rocketmq/connect/jdbc/{ => config}/Config.java | 119 +++++-------
.../rocketmq/connect/jdbc/config/ConfigUtil.java | 52 +++++
.../connect/jdbc/connector/JdbcSinkConnector.java | 58 ++++++
.../connect/jdbc/connector/JdbcSinkTask.java | 133 +++++++++++++
.../jdbc/connector/JdbcSourceConnector.java | 2 +-
.../connect/jdbc/connector/JdbcSourceTask.java | 62 +++---
.../rocketmq/connect/jdbc/schema/Database.java | 43 +++--
.../rocketmq/connect/jdbc/schema/Schema.java | 51 +++--
.../jdbc/schema/column/DateTimeColumnParser.java | 2 +-
.../apache/rocketmq/connect/jdbc/sink/Updater.java | 196 +++++++++++++++++++
.../rocketmq/connect/jdbc/source/JdbcUtils.java | 198 -------------------
.../rocketmq/connect/jdbc/source/Querier.java | 186 ++++++------------
.../jdbc/source/TimestampIncrementingQuerier.java | 24 ++-
14 files changed, 854 insertions(+), 483 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java b/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
new file mode 100644
index 0000000..ccee96b
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
@@ -0,0 +1,211 @@
+
+/**
+ * Copyright 2015 Confluent Inc.
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+package org.apache.rocketmq.connect.jdbc.common;
+
+import com.alibaba.druid.pool.DruidDataSourceFactory;
+import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.*;
+import java.text.SimpleDateFormat;
+import java.util.*;
+import java.util.Date;
+
+/**
+ * Utilties for interacting with a JDBC database.
+ */
+public class DBUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(JdbcSourceTask.class);
+
+ /**
+ * The default table types to include when listing tables if none are specified. Valid values
+ * are those specified by the @{java.sql.DatabaseMetaData#getTables} method's TABLE_TYPE column.
+ * The default only includes standard, user-defined tables.
+ */
+ public static final Set<String> DEFAULT_TABLE_TYPES = Collections.unmodifiableSet(
+ new HashSet<String>(Arrays.asList("TABLE"))
+ );
+
+ private static final int GET_TABLES_TYPE_COLUMN = 4;
+ private static final int GET_TABLES_NAME_COLUMN = 3;
+
+ private static final int GET_COLUMNS_COLUMN_NAME = 4;
+ private static final int GET_COLUMNS_IS_NULLABLE = 18;
+ private static final int GET_COLUMNS_IS_AUTOINCREMENT = 23;
+
+
+ private static ThreadLocal<SimpleDateFormat> DATE_FORMATTER = new ThreadLocal<SimpleDateFormat>() {
+ @Override
+ protected SimpleDateFormat initialValue() {
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return sdf;
+ }
+ };
+
+ /**
+ * Get a list of tables in the database. This uses the default filters, which only include
+ * user-defined tables.
+ * @param conn database connection
+ * @return a list of tables
+ * @throws SQLException
+ */
+ public static List<String> getTables(Connection conn) throws SQLException {
+ return getTables(conn, DEFAULT_TABLE_TYPES);
+ }
+
+ /**
+ * Get a list of table names in the database.
+ * @param conn database connection
+ * @param types a set of table types that should be included in the results
+ * @throws SQLException
+ */
+ public static List<String> getTables(Connection conn, Set<String> types) throws SQLException {
+ DatabaseMetaData metadata = conn.getMetaData();
+ ResultSet rs = metadata.getTables(null, null, "%", null);
+ List<String> tableNames = new ArrayList<String>();
+ while (rs.next()) {
+ if (types.contains(rs.getString(GET_TABLES_TYPE_COLUMN))) {
+ String colName = rs.getString(GET_TABLES_NAME_COLUMN);
+ // SQLite JDBC driver does not correctly mark these as system tables
+ if (metadata.getDatabaseProductName().equals("SQLite") && colName.startsWith("sqlite_")) {
+ continue;
+ }
+
+ tableNames.add(colName);
+ }
+ }
+ return tableNames;
+ }
+
+ /**
+ * Look up the autoincrement column for the specified table.
+ * @param conn database connection
+ * @param table the table to
+ * @return the name of the column that is an autoincrement column, or null if there is no
+ * autoincrement column or more than one exists
+ * @throws SQLException
+ */
+ public static String getAutoincrementColumn(Connection conn, String table) throws SQLException {
+ String result = null;
+ int matches = 0;
+
+ ResultSet rs = conn.getMetaData().getColumns(null, null, table, "%");
+ // Some database drivers (SQLite) don't include all the columns
+ if (rs.getMetaData().getColumnCount() >= GET_COLUMNS_IS_AUTOINCREMENT) {
+ while (rs.next()) {
+ if (rs.getString(GET_COLUMNS_IS_AUTOINCREMENT).equals("YES")) {
+ result = rs.getString(GET_COLUMNS_COLUMN_NAME);
+ matches++;
+ }
+ }
+ return (matches == 1 ? result : null);
+ }
+
+ // Fallback approach is to query for a single row. This unfortunately does not work with any
+ // empty table
+ log.trace("Falling back to SELECT detection of auto-increment column for {}:{}", conn, table);
+ Statement stmt = conn.createStatement();
+ try {
+ String quoteString = getIdentifierQuoteString(conn);
+ rs = stmt.executeQuery("SELECT * FROM " + quoteString + table + quoteString + " LIMIT 1");
+ ResultSetMetaData rsmd = rs.getMetaData();
+ for (int i = 1; i < rsmd.getColumnCount(); i++) {
+ if (rsmd.isAutoIncrement(i)) {
+ result = rsmd.getColumnName(i);
+ matches++;
+ }
+ }
+ } finally {
+ rs.close();
+ stmt.close();
+ }
+ return (matches == 1 ? result : null);
+ }
+
+ public static boolean isColumnNullable(Connection conn, String table, String column)
+ throws SQLException {
+ ResultSet rs = conn.getMetaData().getColumns(null, null, table, column);
+ if (rs.getMetaData().getColumnCount() > GET_COLUMNS_IS_NULLABLE) {
+ // Should only be one match
+ if (!rs.next()) {
+ return false;
+ }
+ String val = rs.getString(GET_COLUMNS_IS_NULLABLE);
+ return rs.getString(GET_COLUMNS_IS_NULLABLE).equals("YES");
+ }
+
+ return false;
+ }
+
+ /**
+ * Format the given Date assuming UTC timezone in a format supported by SQL.
+ * @param date the date to convert to a String
+ * @return the formatted string
+ */
+ public static String formatUTC(Date date) {
+ return DATE_FORMATTER.get().format(date);
+ }
+
+ /**
+ * Get the string used for quoting identifiers in this database's SQL dialect.
+ * @param connection the database connection
+ * @return the quote string
+ * @throws SQLException
+ */
+ public static String getIdentifierQuoteString(Connection connection) throws SQLException {
+ String quoteString = connection.getMetaData().getIdentifierQuoteString();
+ quoteString = quoteString == null ? "" : quoteString;
+ return quoteString;
+ }
+
+ /**
+ * Quote the given string.
+ * @param orig the string to quote
+ * @param quote the quote character
+ * @return the quoted string
+ */
+ public static String quoteString(String orig, String quote) {
+ return quote + orig + quote;
+ }
+
+ public static DataSource initDataSource(Config config) throws Exception {
+ Map<String, String> map = new HashMap<>();
+ map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
+ map.put("url",
+ "jdbc:mysql://" + config.getJdbcUrl() + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8&characterEncoding=utf8");
+ map.put("username", config.getJdbcUsername());
+ map.put("password", config.getJdbcPassword());
+ map.put("initialSize", "1");
+ map.put("maxActive", "1");
+ map.put("maxWait", "60000");
+ map.put("timeBetweenEvictionRunsMillis", "60000");
+ map.put("minEvictableIdleTimeMillis", "300000");
+ map.put("validationQuery", "SELECT 1 FROM DUAL");
+ map.put("testWhileIdle", "true");
+ log.info("{} config read successful", map);
+ DataSource dataSource = DruidDataSourceFactory.createDataSource(map);
+ log.info("init data source success");
+ return dataSource;
+ }
+}
+
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
similarity index 65%
rename from src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
rename to src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index f93c4db..5491d43 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
@@ -15,13 +15,11 @@
* limitations under the License.
*/
-package org.apache.rocketmq.connect.jdbc;
+package org.apache.rocketmq.connect.jdbc.config;
-import io.openmessaging.KeyValue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.lang.reflect.Method;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -30,35 +28,37 @@ public class Config {
private static final Logger LOG = LoggerFactory.getLogger(Config.class);
/* Database Connection Config */
- public String jdbcUrl = "localhost:3306";
- public String jdbcUsername = "root";
- public String jdbcPassword = "199812160";
- public String rocketmqTopic;
- public String jdbcBackoff;
- public String jdbcAttempts;
- public String catalogPattern = null;
- public List tableWhitelist;
- public List tableBlacklist;
- public String schemaPattern = null;
- public boolean numericPrecisionMapping = false;
- public String bumericMapping = null;
- public String dialectName = "";
+ private String jdbcUrl;
+ private String jdbcUsername;
+ private String jdbcPassword;
+ private String rocketmqTopic;
+ private String jdbcBackoff;
+ private String jdbcAttempts;
+ private String catalogPattern;
+ private List tableWhitelist;
+ private List tableBlacklist;
+ private String schemaPattern;
+ private boolean numericPrecisionMapping = false;
+ private String bumericMapping;
+ private String dialectName = "";
+ private String whiteDataBase;
+ private String whiteTable;
/* Mode Config */
- public String mode = "";
- public String incrementingColumnName = "";
- public String query = "";
- public String timestampColmnName = "";
- public boolean validateNonNull = true;
+ private String mode = "";
+ private String incrementingColumnName = "";
+ private String query = "";
+ private String timestampColmnName = "";
+ private boolean validateNonNull = true;
/*Connector config*/
- public String tableTypes = "table";
- public long pollInterval = 5000;
- public int batchMaxRows = 100;
- public long tablePollInterval = 60000;
- public long timestampDelayInterval = 0;
- public String dbTimezone = "UTC";
- public String queueName;
+ private String tableTypes = "table";
+ private long pollInterval = 5000;
+ private int batchMaxRows = 100;
+ private long tablePollInterval = 60000;
+ private long timestampDelayInterval = 0;
+ private String dbTimezone = "UTC";
+ private String queueName;
private Logger log = LoggerFactory.getLogger(Config.class);
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
@@ -71,53 +71,6 @@ public class Config {
}
};
-
- public void load(KeyValue props) {
- log.info("Config.load.start");
- properties2Object(props, this);
- }
-
- private void properties2Object(final KeyValue p, final Object object) {
- Method[] methods = object.getClass().getMethods();
- for (Method method : methods) {
- String mn = method.getName();
- if (mn.startsWith("set")) {
- try {
- String tmp = mn.substring(4);
- String first = mn.substring(3, 4);
-
- String key = first.toLowerCase() + tmp;
- String property = p.getString(key);
- if (property != null) {
- Class<?>[] pt = method.getParameterTypes();
- if (pt != null && pt.length > 0) {
- String cn = pt[0].getSimpleName();
- Object arg = null;
- if (cn.equals("int") || cn.equals("Integer")) {
- arg = Integer.parseInt(property);
- } else if (cn.equals("long") || cn.equals("Long")) {
- arg = Long.parseLong(property);
- } else if (cn.equals("double") || cn.equals("Double")) {
- arg = Double.parseDouble(property);
- } else if (cn.equals("boolean") || cn.equals("Boolean")) {
- arg = Boolean.parseBoolean(property);
- } else if (cn.equals("float") || cn.equals("Float")) {
- arg = Float.parseFloat(property);
- } else if (cn.equals("String")) {
- arg = property;
- } else {
- continue;
- }
- method.invoke(object, arg);
-
- }
- }
- } catch (Throwable ignored) {
- }
- }
- }
- }
-
public String getQueueName() {
return queueName;
}
@@ -317,4 +270,20 @@ public class Config {
public void setDbTimezone(String dbTimezone) {
this.dbTimezone = dbTimezone;
}
+
+ public String getWhiteDataBase() {
+ return whiteDataBase;
+ }
+
+ public void setWhiteDataBase(String whiteDataBase) {
+ this.whiteDataBase = whiteDataBase;
+ }
+
+ public String getWhiteTable() {
+ return whiteTable;
+ }
+
+ public void setWhiteTable(String whiteTable) {
+ this.whiteTable = whiteTable;
+ }
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigUtil.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigUtil.java
new file mode 100644
index 0000000..53563f2
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigUtil.java
@@ -0,0 +1,52 @@
+package org.apache.rocketmq.connect.jdbc.config;
+
+import io.openmessaging.KeyValue;
+import java.lang.reflect.Method;
+
+public class ConfigUtil {
+ public static <T> void load(KeyValue props, Object object) {
+
+ properties2Object(props, object);
+ }
+
+ private static <T> void properties2Object(final KeyValue p, final Object object) {
+
+ Method[] methods = object.getClass().getMethods();
+ for (Method method : methods) {
+ String mn = method.getName();
+ if (mn.startsWith("set")) {
+ try {
+ String tmp = mn.substring(4);
+ String first = mn.substring(3, 4);
+
+ String key = first.toLowerCase() + tmp;
+ String property = p.getString(key);
+ if (property != null) {
+ Class<?>[] pt = method.getParameterTypes();
+ if (pt != null && pt.length > 0) {
+ String cn = pt[0].getSimpleName();
+ Object arg;
+ if (cn.equals("int") || cn.equals("Integer")) {
+ arg = Integer.parseInt(property);
+ } else if (cn.equals("long") || cn.equals("Long")) {
+ arg = Long.parseLong(property);
+ } else if (cn.equals("double") || cn.equals("Double")) {
+ arg = Double.parseDouble(property);
+ } else if (cn.equals("boolean") || cn.equals("Boolean")) {
+ arg = Boolean.parseBoolean(property);
+ } else if (cn.equals("float") || cn.equals("Float")) {
+ arg = Float.parseFloat(property);
+ } else if (cn.equals("String")) {
+ arg = property;
+ } else {
+ continue;
+ }
+ method.invoke(object, arg);
+ }
+ }
+ } catch (Throwable ignored) {
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
new file mode 100644
index 0000000..e1d1256
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
@@ -0,0 +1,58 @@
+package org.apache.rocketmq.connect.jdbc.connector;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import io.openmessaging.connector.api.sink.SinkConnector;
+import org.apache.rocketmq.connect.jdbc.config.Config;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+public class JdbcSinkConnector extends SinkConnector {
+
+ private KeyValue config;
+
+ @Override
+ public String verifyAndSetConfig(KeyValue config) {
+ for (String requestKey : Config.REQUEST_CONFIG) {
+ if (!config.containsKey(requestKey)) {
+ return "Request config key: " + requestKey;
+ }
+ }
+ this.config = config;
+ return "";
+ }
+
+ @Override
+ public void start() {
+
+ }
+
+ @Override
+ public void stop() {
+
+ }
+
+ @Override
+ public void pause() {
+
+ }
+
+ @Override
+ public void resume() {
+
+ }
+
+ @Override
+ public Class<? extends Task> taskClass() {
+ return JdbcSinkTask.class;
+ }
+
+ @Override
+ public List<KeyValue> taskConfigs() {
+ List<KeyValue> config = new ArrayList<>();
+ config.add(this.config);
+ return config;
+ }
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java
new file mode 100644
index 0000000..4b55e5a
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java
@@ -0,0 +1,133 @@
+package org.apache.rocketmq.connect.jdbc.connector;
+
+
+import com.alibaba.fastjson.JSONObject;
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.common.QueueMetaData;
+import io.openmessaging.connector.api.data.EntryType;
+import io.openmessaging.connector.api.data.Field;
+import io.openmessaging.connector.api.data.Schema;
+import io.openmessaging.connector.api.data.SinkDataEntry;
+import io.openmessaging.connector.api.sink.SinkTask;
+import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.common.DBUtils;
+import org.apache.rocketmq.connect.jdbc.config.ConfigUtil;
+import org.apache.rocketmq.connect.jdbc.sink.Updater;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.Connection;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+public class JdbcSinkTask extends SinkTask {
+
+ private static final Logger log = LoggerFactory.getLogger(JdbcSinkTask.class);
+
+ private Config config;
+ private DataSource dataSource;
+ private Connection connection;
+ private Updater updater;
+ private BlockingQueue<Updater> tableQueue = new LinkedBlockingQueue<Updater>();
+
+ public JdbcSinkTask() {
+ this.config = new Config();
+ }
+
+ @Override
+ public void put(Collection<SinkDataEntry> sinkDataEntries) {
+ try {
+ if (tableQueue.size() > 1) {
+ updater = tableQueue.poll(1000, TimeUnit.MILLISECONDS);
+ } else {
+ updater = tableQueue.peek();
+ }
+
+ for (SinkDataEntry record : sinkDataEntries) {
+ Map<Field, Object[]> fieldMap = new HashMap<>();
+ Object[] payloads = record.getPayload();
+ Schema schema = record.getSchema();
+ EntryType entryType = record.getEntryType();
+ String tableName = schema.getName();
+ String dbName = schema.getDataSource();
+ List<Field> fields = schema.getFields();
+ Boolean parseError = false;
+ if (!fields.isEmpty()) {
+ for (Field field : fields) {
+ Object fieldValue = payloads[field.getIndex()];
+ Object[] value = JSONObject.parseArray((String)fieldValue).toArray();
+ if (value.length == 2) {
+ fieldMap.put(field, value);
+ } else {
+ log.error("parseArray error, fieldValue:{}", fieldValue);
+ parseError = true;
+ }
+ }
+ }
+ if (!parseError) {
+ Boolean isSuccess = updater.push(dbName, tableName, fieldMap, entryType);
+ if (!isSuccess) {
+ log.error("push data error, dbName:{}, tableName:{}, entryType:{}, fieldMap:{}", dbName, tableName, fieldMap, entryType);
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("put sinkDataEntries error, {}", e);
+ }
+ }
+
+ @Override
+ public void commit(Map<QueueMetaData, Long> map) {
+
+ }
+
+ @Override
+ public void start(KeyValue props) {
+ try {
+ ConfigUtil.load(props, this.config);
+ dataSource = DBUtils.initDataSource(config);
+ connection = dataSource.getConnection();
+ log.info("init data source success");
+ } catch (Exception e) {
+ log.error("Cannot start Jdbc Sink Task because of configuration error{}", e);
+ }
+ String mode = config.getMode();
+ if (mode.equals("bulk")) {
+ Updater updater = new Updater(config, connection);
+ try {
+ updater.start();
+ tableQueue.add(updater);
+ } catch (Exception e) {
+ log.error("fail to start updater{}", e);
+ }
+ }
+ }
+
+ @Override
+ public void stop() {
+ try {
+ if (connection != null){
+ connection.close();
+ }
+ } catch (Throwable e) {
+ log.warn("sink task stop error while closing connection to {}", "jdbc", e);
+ }
+ }
+
+ @Override
+ public void pause() {
+
+ }
+
+ @Override
+ public void resume() {
+
+ }
+
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
index 4a870c0..796f0e6 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
@@ -20,7 +20,7 @@ package org.apache.rocketmq.connect.jdbc.connector;
import java.util.ArrayList;
import java.util.List;
-import org.apache.rocketmq.connect.jdbc.Config;
+import org.apache.rocketmq.connect.jdbc.config.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index 767c3aa..943d432 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -21,19 +21,19 @@ package org.apache.rocketmq.connect.jdbc.connector;
import io.openmessaging.connector.api.source.SourceTask;
import java.nio.ByteBuffer;
-import java.sql.SQLException;
+import java.sql.Connection;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Timer;
-import java.util.TimerTask;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
-import org.apache.rocketmq.connect.jdbc.Config;
+import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.common.DBUtils;
+import org.apache.rocketmq.connect.jdbc.config.ConfigUtil;
import org.apache.rocketmq.connect.jdbc.schema.Table;
import org.apache.rocketmq.connect.jdbc.source.Querier;
import org.apache.rocketmq.connect.jdbc.source.TimestampIncrementingQuerier;
@@ -53,17 +53,27 @@ import com.alibaba.fastjson.JSONObject;
import io.openmessaging.connector.api.data.DataEntryBuilder;
import io.openmessaging.connector.api.data.Field;
+import javax.sql.DataSource;
+
public class JdbcSourceTask extends SourceTask {
private static final Logger log = LoggerFactory.getLogger(JdbcSourceTask.class);
private Config config;
+ private DataSource dataSource;
+
+ private Connection connection;
+
BlockingQueue<Querier> tableQueue = new LinkedBlockingQueue<Querier>();
static final String INCREMENTING_FIELD = "incrementing";
static final String TIMESTAMP_FIELD = "timestamp";
private Querier querier;
+ public JdbcSourceTask() {
+ this.config = new Config();
+ }
+
@Override
public Collection<SourceDataEntry> poll() {
List<SourceDataEntry> res = new ArrayList<>();
@@ -96,42 +106,45 @@ public class JdbcSourceTask extends SourceTask {
}
DataEntryBuilder dataEntryBuilder = new DataEntryBuilder(schema);
dataEntryBuilder.timestamp(System.currentTimeMillis()).queue(dataRow.getName())
- .entryType(EntryType.CREATE);
+ .entryType(EntryType.UPDATE);
for (int i = 0; i < dataRow.getColList().size(); i++) {
- Object value = dataRow.getDataList().get(i);
- // System.out.println(dataRow.getColList().get(i) + "|" + value);
- dataEntryBuilder.putFiled(dataRow.getColList().get(i), value);
+ Object[] value = new Object[2];
+ value[0] = value[1] = dataRow.getDataList().get(i);
+ dataEntryBuilder.putFiled(dataRow.getColList().get(i), JSONObject.toJSONString(value));
}
SourceDataEntry sourceDataEntry = dataEntryBuilder.buildSourceDataEntry(
- ByteBuffer.wrap(config.jdbcUrl.getBytes("UTF-8")),
+ ByteBuffer.wrap(config.getJdbcUrl().getBytes("UTF-8")),
ByteBuffer.wrap(jsonObject.toJSONString().getBytes("UTF-8")));
res.add(sourceDataEntry);
-
+ log.debug("sourceDataEntry : {}", JSONObject.toJSONString(sourceDataEntry));
}
} catch (Exception e) {
log.error("JDBC task poll error, current config:" + JSON.toJSONString(config), e);
}
- log.info("dataEntry poll successfully,{}", res);
+ log.debug("dataEntry poll successfully,{}", JSONObject.toJSONString(res));
return res;
}
@Override
public void start(KeyValue props) {
- config = new Config();
- config.load(props);
-
+ try {
+ ConfigUtil.load(props, this.config);
+ dataSource = DBUtils.initDataSource(config);
+ connection = dataSource.getConnection();
+ log.info("init data source success");
+ } catch (Exception e) {
+ log.error("Cannot start Jdbc Source Task because of configuration error{}", e);
+ }
Map<Map<String, String>, Map<String, Object>> offsets = null;
- String mode = config.mode;
+ String mode = config.getMode();
if (mode.equals("bulk")) {
- Querier querier = new Querier();
+ Querier querier = new Querier(config, connection);
try {
- querier.setConfig(config);
querier.start();
tableQueue.add(querier);
} catch (Exception e) {
- // TODO Auto-generated catch block
- log.error("Start unsuccessfully Because of {}",e);
+ log.error("start querier failed in bulk mode{}", e);
}
} else {
TimestampIncrementingQuerier querier = new TimestampIncrementingQuerier();
@@ -140,8 +153,7 @@ public class JdbcSourceTask extends SourceTask {
querier.start();
tableQueue.add(querier);
} catch (Exception e) {
- // TODO Auto-generated catch block
- log.error("Start unsuccessfully Because of {}",e);
+ log.error("fail to start querier{}", e);
}
}
@@ -150,7 +162,13 @@ public class JdbcSourceTask extends SourceTask {
@Override
public void stop() {
- querier.stop();
+ try {
+ if (connection != null){
+ connection.close();
+ }
+ } catch (Throwable e) {
+ log.warn("source task stop error while closing connection to {}", "jdbc", e);
+ }
}
@Override
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
index 657ebb8..49e28cd 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
@@ -17,41 +17,41 @@
package org.apache.rocketmq.connect.jdbc.schema;
-//import io.openmessaging.mysql.binlog.EventProcessor;
-
import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-import javax.sql.DataSource;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Database {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Database.class);
+
private static final String SQL = "select table_name,column_name,data_type,column_type,character_set_name " +
"from information_schema.columns " +
"where table_schema = ? order by ORDINAL_POSITION";
private String name;
- private DataSource dataSource;
- public Map<String, Table> tableMap = new HashMap<String, Table>();
- public Database(String name, DataSource dataSource) {
+ private Connection connection;
+
+ private Map<String, Table> tableMap = new HashMap<String, Table>();
+
+ public Set<String> tableWhiteList;
+
+ public Database(String name, Connection connection, Set<String> tableWhiteList) {
this.name = name;
- this.dataSource = dataSource;
+ this.connection = connection;
+ this.tableWhiteList = tableWhiteList;
}
public void init() throws SQLException {
- Connection conn = null;
PreparedStatement ps = null;
ResultSet rs = null;
try {
- conn = dataSource.getConnection();
-
- ps = conn.prepareStatement(SQL);
+ ps = connection.prepareStatement(SQL);
ps.setString(1, name);
rs = ps.executeQuery();
@@ -63,7 +63,9 @@ public class Database {
String charset = rs.getString(5);
ColumnParser columnParser = ColumnParser.getColumnParser(dataType, colType, charset);
-
+ if (!tableWhiteList.contains(tableName)){
+ continue;
+ }
if (!tableMap.containsKey(tableName)) {
addTable(tableName);
}
@@ -74,21 +76,20 @@ public class Database {
}
} finally {
- if (conn != null) {
- conn.close();
+ if (rs != null) {
+ rs.close();
}
if (ps != null) {
ps.close();
}
- if (rs != null) {
- rs.close();
- }
}
}
private void addTable(String tableName) {
+ LOGGER.info("Schema load -- DATABASE:{},\tTABLE:{}", name, tableName);
+
Table table = new Table(name, tableName);
tableMap.put(tableName, table);
}
@@ -97,4 +98,8 @@ public class Database {
return tableMap.get(tableName);
}
+
+ public Map<String, Table> getTableMap() {
+ return tableMap;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
index 93b204f..16d636f 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
@@ -21,65 +21,59 @@ import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import javax.sql.DataSource;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Schema {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Schema.class);
private static final String SQL = "select schema_name from information_schema.schemata";
- //acquiring databases
+
private static final List<String> IGNORED_DATABASES = new ArrayList<>(
Arrays.asList(new String[] {"information_schema", "mysql", "performance_schema", "sys"})
);
- //ignored databases (System Databases)
- private DataSource dataSource;
- public Map<String, Database> dbMap;
+ public Set<String> dataBaseWhiteList;
+
+ public Set<String> tableWhiteList;
+
+ private Connection connection;
+
+ private Map<String, Database> dbMap;
- public Schema(DataSource dataSource) {
- this.dataSource = dataSource;
+ public Schema(Connection connection) {
+ this.connection = connection;
+ this.dataBaseWhiteList = new HashSet<>();
+ this.tableWhiteList = new HashSet<>();
}
public void load() throws SQLException {
dbMap = new HashMap<>();
- Connection conn = null;
PreparedStatement ps = null;
ResultSet rs = null;
try {
- conn = dataSource.getConnection();
-
- ps = conn.prepareStatement(SQL);
+ ps = connection.prepareStatement(SQL);
rs = ps.executeQuery();
while (rs.next()) {
String dbName = rs.getString(1);
- if (!IGNORED_DATABASES.contains(dbName)) {
- Database database = new Database(dbName, dataSource);
+ if (!IGNORED_DATABASES.contains(dbName) && dataBaseWhiteList.contains(dbName)) {
+ Database database = new Database(dbName, connection, tableWhiteList);
dbMap.put(dbName, database);
- //dbMap存着各个数据库的名字
}
}
} finally {
-
- if (conn != null) {
- conn.close();
+ if (rs != null) {
+ rs.close();
}
if (ps != null) {
ps.close();
}
- if (rs != null) {
- rs.close();
- }
}
for (Database db : dbMap.values()) {
@@ -114,8 +108,7 @@ public class Schema {
load();
break;
} catch (Exception e) {
- // LOGGER.error("Reload schema error.", e);
- System.out.println("Reload schema error." + e);
+ LOGGER.error("Reload schema error.", e);
}
}
}
@@ -123,4 +116,8 @@ public class Schema {
public void reset() {
dbMap = null;
}
+
+ public Map<String, Database> getDbMap() {
+ return dbMap;
+ }
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java
index 8736280..c9b39e3 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java
@@ -30,7 +30,7 @@ public class DateTimeColumnParser extends ColumnParser {
static {
dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
dateTimeUtcFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeUtcFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ dateTimeUtcFormat.setTimeZone(TimeZone.getTimeZone("GMT+8"));
}
@Override
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java b/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
new file mode 100644
index 0000000..3571852
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
@@ -0,0 +1,196 @@
+package org.apache.rocketmq.connect.jdbc.sink;
+
+import io.openmessaging.connector.api.data.EntryType;
+import io.openmessaging.connector.api.data.Field;
+import io.openmessaging.connector.api.data.FieldType;
+import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.schema.Schema;
+import org.apache.rocketmq.connect.jdbc.schema.column.DateTimeColumnParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+public class Updater {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ private final Queue<Connection> connections = new ConcurrentLinkedQueue<>();
+ private Config config;
+ private Schema schema;
+ private Connection connection;
+
+ public Updater(Config config, Connection connection) {
+ this.config = config;
+ this.connection = connection;
+ this.schema = new Schema(connection);
+ }
+
+ public boolean push(String dbName, String tableName, Map<Field, Object[]> fieldMap, EntryType entryType) {
+ Boolean isSuccess = false;
+ int id = 0;
+ switch (entryType) {
+ case CREATE:
+ isSuccess = updateRow(dbName, tableName, fieldMap, id);
+ break;
+ case UPDATE:
+ id = queryRowId(dbName, tableName, fieldMap);
+ isSuccess = updateRow(dbName, tableName, fieldMap, id);
+ break;
+ case DELETE:
+ id = queryRowId(dbName, tableName, fieldMap);
+ isSuccess = deleteRow(dbName, tableName, id);
+ break;
+ default:
+ log.error("entryType {} is illegal.", entryType.toString());
+ }
+ return isSuccess;
+ }
+
+ public void start() throws Exception {
+ schema.load();
+ log.info("schema load success");
+ }
+
+ public Config getConfig() {
+ return config;
+ }
+
+ public void setConfig(Config config) {
+ this.config = config;
+ }
+
+ private String typeParser(FieldType fieldType, String fieldName, Object fieldValue, String sql) {
+ switch (fieldType) {
+ case STRING:
+ sql += fieldName + " = " + "'" + fieldValue + "'";
+ break;
+ case DATETIME:
+ sql += fieldName + " = " + "'" + new DateTimeColumnParser().getValue(fieldValue) + "'";
+ break;
+ case INT32:
+ case INT64:
+ case FLOAT32:
+ case FLOAT64:
+ case BIG_INTEGER:
+ sql += fieldName + " = " + fieldValue;
+ break;
+ default:
+ log.error("fieldType {} is illegal.", fieldType.toString());
+ }
+ return sql;
+ }
+
+ private Integer queryRowId(String dbName, String tableName, Map<Field, Object[]> fieldMap) {
+ int count = 0, id = 0;
+ ResultSet rs;
+ PreparedStatement stmt;
+ Boolean finishQuery = false;
+ String query = "select id from " + dbName + "." + tableName + " where ";
+
+ for (Map.Entry<Field, Object[]> entry : fieldMap.entrySet()) {
+ count ++;
+ String fieldName = entry.getKey().getName();
+ FieldType fieldType = entry.getKey().getType();
+ Object fieldValue = entry.getValue()[0];
+ if ("id".equals(fieldName))
+ continue;
+ if (count != 1) {
+ query += " and ";
+ }
+ if (fieldValue == null)
+ {
+ query += fieldName + " is NULL";
+ } else {
+ query = typeParser(fieldType, fieldName, fieldValue, query);
+ }
+ }
+
+ try {
+ while (!connection.isClosed() && !finishQuery){
+ stmt = connection.prepareStatement(query);
+ rs = stmt.executeQuery();
+ if (rs != null) {
+ while (rs.next()) {
+ id = rs.getInt("id");
+ }
+ finishQuery = true;
+ rs.close();
+ }
+ }
+ } catch (SQLException e) {
+ log.error("query table error,{}", e);
+ }
+ return id;
+ }
+
+ private Boolean updateRow(String dbName, String tableName, Map<Field, Object[]> fieldMap, Integer id) {
+ int count = 0;
+ PreparedStatement stmt;
+ boolean finishUpdate = false;
+ String update = "replace into " + dbName + "." + tableName + " set ";
+
+ for (Map.Entry<Field, Object[]> entry : fieldMap.entrySet()) {
+ count++;
+ String fieldName = entry.getKey().getName();
+ FieldType fieldType = entry.getKey().getType();
+ Object fieldValue = entry.getValue()[1];
+ if ("id".equals(fieldName)) {
+ if (id == 0)
+ continue;
+ else
+ fieldValue = id;
+ }
+ if (count != 1) {
+ update += ", ";
+ }
+ if (fieldValue == null) {
+ update += fieldName + " = NULL";
+ } else {
+ update = typeParser(fieldType, fieldName, fieldValue, update);
+ }
+ }
+
+ try {
+ while (!connection.isClosed() && !finishUpdate){
+ stmt = connection.prepareStatement(update);
+ int result = stmt.executeUpdate();
+ if (result > 0) {
+ log.info("replace into table success");
+ return true;
+ }
+ finishUpdate = true;
+ stmt.close();
+ }
+ } catch (SQLException e) {
+ log.error("update table error,{}", e);
+ }
+ return false;
+ }
+
+ private Boolean deleteRow(String dbName, String tableName, Integer id) {
+ PreparedStatement stmt;
+ String delete = "delete from " + dbName + "." + tableName + " where id = " + id ;
+ boolean finishDelete = false;
+ try {
+ while (!connection.isClosed() && !finishDelete){
+ stmt = connection.prepareStatement(delete);
+ int result = stmt.executeUpdate();
+ if (result > 0) {
+ log.info("delete from table success");
+ return true;
+ }
+ finishDelete = true;
+ stmt.close();
+ }
+ } catch (SQLException e) {
+ log.error("delete from table error,{}", e);
+ }
+ return false;
+ }
+
+}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcUtils.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcUtils.java
deleted file mode 100644
index cbcca6a..0000000
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/JdbcUtils.java
+++ /dev/null
@@ -1,198 +0,0 @@
-
-/**
- * Copyright 2015 Confluent Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- **/
-
-package org.apache.rocketmq.connect.jdbc.source;
-import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.Connection;
-import java.sql.DatabaseMetaData;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.TimeZone;
-
-/**
- * Utilties for interacting with a JDBC database.
- */
-public class JdbcUtils {
-
- private static final Logger log = LoggerFactory.getLogger(JdbcSourceTask.class);
-
- /**
- * The default table types to include when listing tables if none are specified. Valid values
- * are those specified by the @{java.sql.DatabaseMetaData#getTables} method's TABLE_TYPE column.
- * The default only includes standard, user-defined tables.
- */
- public static final Set<String> DEFAULT_TABLE_TYPES = Collections.unmodifiableSet(
- new HashSet<String>(Arrays.asList("TABLE"))
- );
-
- private static final int GET_TABLES_TYPE_COLUMN = 4;
- private static final int GET_TABLES_NAME_COLUMN = 3;
-
- private static final int GET_COLUMNS_COLUMN_NAME = 4;
- private static final int GET_COLUMNS_IS_NULLABLE = 18;
- private static final int GET_COLUMNS_IS_AUTOINCREMENT = 23;
-
-
- private static ThreadLocal<SimpleDateFormat> DATE_FORMATTER = new ThreadLocal<SimpleDateFormat>() {
- @Override
- protected SimpleDateFormat initialValue() {
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
- sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
- return sdf;
- }
- };
-
- /**
- * Get a list of tables in the database. This uses the default filters, which only include
- * user-defined tables.
- * @param conn database connection
- * @return a list of tables
- * @throws SQLException
- */
- public static List<String> getTables(Connection conn) throws SQLException {
- return getTables(conn, DEFAULT_TABLE_TYPES);
- }
-
- /**
- * Get a list of table names in the database.
- * @param conn database connection
- * @param types a set of table types that should be included in the results
- * @throws SQLException
- */
- public static List<String> getTables(Connection conn, Set<String> types) throws SQLException {
- DatabaseMetaData metadata = conn.getMetaData();
- ResultSet rs = metadata.getTables(null, null, "%", null);
- List<String> tableNames = new ArrayList<String>();
- while (rs.next()) {
- if (types.contains(rs.getString(GET_TABLES_TYPE_COLUMN))) {
- String colName = rs.getString(GET_TABLES_NAME_COLUMN);
- // SQLite JDBC driver does not correctly mark these as system tables
- if (metadata.getDatabaseProductName().equals("SQLite") && colName.startsWith("sqlite_")) {
- continue;
- }
-
- tableNames.add(colName);
- }
- }
- return tableNames;
- }
-
- /**
- * Look up the autoincrement column for the specified table.
- * @param conn database connection
- * @param table the table to
- * @return the name of the column that is an autoincrement column, or null if there is no
- * autoincrement column or more than one exists
- * @throws SQLException
- */
- public static String getAutoincrementColumn(Connection conn, String table) throws SQLException {
- String result = null;
- int matches = 0;
-
- ResultSet rs = conn.getMetaData().getColumns(null, null, table, "%");
- // Some database drivers (SQLite) don't include all the columns
- if (rs.getMetaData().getColumnCount() >= GET_COLUMNS_IS_AUTOINCREMENT) {
- while(rs.next()) {
- if (rs.getString(GET_COLUMNS_IS_AUTOINCREMENT).equals("YES")) {
- result = rs.getString(GET_COLUMNS_COLUMN_NAME);
- matches++;
- }
- }
- return (matches == 1 ? result : null);
- }
-
- // Fallback approach is to query for a single row. This unfortunately does not work with any
- // empty table
- log.trace("Falling back to SELECT detection of auto-increment column for {}:{}", conn, table);
- Statement stmt = conn.createStatement();
- try {
- String quoteString = getIdentifierQuoteString(conn);
- rs = stmt.executeQuery("SELECT * FROM " + quoteString + table + quoteString + " LIMIT 1");
- ResultSetMetaData rsmd = rs.getMetaData();
- for(int i = 1; i < rsmd.getColumnCount(); i++) {
- if (rsmd.isAutoIncrement(i)) {
- result = rsmd.getColumnName(i);
- matches++;
- }
- }
- } finally {
- rs.close();
- stmt.close();
- }
- return (matches == 1 ? result : null);
- }
-
- public static boolean isColumnNullable(Connection conn, String table, String column)
- throws SQLException {
- ResultSet rs = conn.getMetaData().getColumns(null, null, table, column);
- if (rs.getMetaData().getColumnCount() > GET_COLUMNS_IS_NULLABLE) {
- // Should only be one match
- if (!rs.next()) {
- return false;
- }
- String val = rs.getString(GET_COLUMNS_IS_NULLABLE);
- return rs.getString(GET_COLUMNS_IS_NULLABLE).equals("YES");
- }
-
- return false;
- }
-
- /**
- * Format the given Date assuming UTC timezone in a format supported by SQL.
- * @param date the date to convert to a String
- * @return the formatted string
- */
- public static String formatUTC(Date date) {
- return DATE_FORMATTER.get().format(date);
- }
-
- /**
- * Get the string used for quoting identifiers in this database's SQL dialect.
- * @param connection the database connection
- * @return the quote string
- * @throws SQLException
- */
- public static String getIdentifierQuoteString(Connection connection) throws SQLException {
- String quoteString = connection.getMetaData().getIdentifierQuoteString();
- quoteString = quoteString == null ? "" : quoteString;
- return quoteString;
- }
-
- /**
- * Quote the given string.
- * @param orig the string to quote
- * @param quote the quote character
- * @return the quoted string
- */
- public static String quoteString(String orig, String quote) {
- return quote + orig + quote;
- }
-}
-
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index 13cbda5..8da3f21 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -1,104 +1,69 @@
package org.apache.rocketmq.connect.jdbc.source;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.schema.Database;
+import org.apache.rocketmq.connect.jdbc.schema.Schema;
+import org.apache.rocketmq.connect.jdbc.schema.Table;
+import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.sql.Connection;
-import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Queue;
+import java.util.*;
import java.util.concurrent.ConcurrentLinkedQueue;
-import javax.sql.DataSource;
-import org.apache.rocketmq.connect.jdbc.Config;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.alibaba.druid.pool.DruidDataSourceFactory;
-import org.apache.rocketmq.connect.jdbc.schema.*;
-import org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser;
public class Querier {
- private final Logger log = LoggerFactory.getLogger(getClass()); // use concrete subclass
+ private final Logger log = LoggerFactory.getLogger(Querier.class); // use concrete subclass
protected String topicPrefix;
protected String jdbcUrl;
private final Queue<Connection> connections = new ConcurrentLinkedQueue<>();
private Config config;
-
- /**
- * @return the config
- */
- public Config getConfig() {
- return config;
- }
-
- public void setConfig(Config config) {
- this.config = config;
- log.info("config load successfully");
- }
-
- private DataSource dataSource;
+ private Connection connection;
private List<Table> list = new LinkedList<>();
private String mode;
+ private Schema schema;
+ public Querier(){
- public DataSource getDataSource() {
- return dataSource;
}
- public void setDataSource(DataSource dataSource) {
- this.dataSource = dataSource;
+ public Querier(Config config, Connection connection) {
+ this.config = config;
+ this.connection = connection;
+ this.schema = new Schema(connection);
}
- public String getMode() {
- return mode;
+ /**
+ * @return the config
+ */
+ public Config getConfig() {
+ return config;
}
- public void setMode(String mode) {
- this.mode = mode;
+ public void setConfig(Config config) {
+ this.config = config;
}
+// public String getMode() {
+// return mode;
+// }
+//
+// public void setMode(String mode) {
+// this.mode = mode;
+// }
public List<Table> getList() {
return list;
}
- public void setList(List<Table> list) {
- this.list = list;
- }
-
- public Connection getConnection() throws SQLException {
- // These config names are the same for both source and sink configs ...
- String username = config.jdbcUsername;
- String dbPassword = config.jdbcPassword;
- jdbcUrl = config.jdbcUrl;
- Properties properties = new Properties();
- if (username != null) {
- properties.setProperty("user", username);
- }
- if (dbPassword != null) {
- properties.setProperty("password", dbPassword);
- }
- Connection connection = DriverManager.getConnection(jdbcUrl, properties);
-
- connections.add(connection);
- return connection;
- }
-
- public void stop() {
- Connection conn;
- while ((conn = connections.poll()) != null) {
- try {
- conn.close();
- } catch (Throwable e) {
- log.warn("Error while closing connection to {}", "jdbc", e);
- }
- }
- }
+// public void setList(List<Table> list) {
+// this.list = list;
+// }
protected PreparedStatement createDBPreparedStatement(Connection db) throws SQLException {
@@ -123,32 +88,18 @@ public class Querier {
return stmt.executeQuery();
}
- private Schema schema;
-
- public static void main(String[] args) throws Exception {
- TimestampIncrementingQuerier querier = new TimestampIncrementingQuerier();
- try {
- querier.start();
- querier.poll();
- } catch (SQLException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
-
- public void poll() {
+ public void poll() {
try {
-
PreparedStatement stmt;
String query = "select * from ";
- Connection conn = dataSource.getConnection();
- for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
+ LinkedList<Table> tableLinkedList = new LinkedList<>();
+ for (Map.Entry<String, Database> entry : schema.getDbMap().entrySet()) {
String db = entry.getKey();
- Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
+ Iterator<Map.Entry<String, Table>> iterator = entry.getValue().getTableMap().entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
String tb = tableEntry.getKey();
- stmt = conn.prepareStatement(query + db + "." + tb);
+ stmt = connection.prepareStatement(query + db + "." + tb);
ResultSet rs;
rs = stmt.executeQuery();
List<String> colList = tableEntry.getValue().getColList();
@@ -157,64 +108,45 @@ public class Querier {
while (rs.next()) {
Table table = new Table(db, tb);
- System.out.print("|");
+ //System.out.print("|");
table.setColList(colList);
table.setRawDataTypeList(DataTypeList);
table.setParserList(ParserList);
for (String string : colList) {
table.getDataList().add(rs.getObject(string));
- System.out.print(string + " : " + rs.getObject(string) + "|");
+ //System.out.print(string + " : " + rs.getObject(string) + "|");
}
- list.add(table);
- System.out.println();
+ tableLinkedList.add(table);
}
+ rs.close();
+ stmt.close();
}
}
-
+ list = tableLinkedList;
} catch (SQLException e) {
- e.printStackTrace();
+ log.error("fail to poll data, {}", e);
}
}
public void start() throws Exception {
- try {
+ String whiteDataBases = config.getWhiteDataBase();
+ String whiteTables = config.getWhiteTable();
- log.info("datasorce success");
- initDataSource();
- } catch (Throwable exception) {
- log.info("error,{}", exception);
+ if (!StringUtils.isEmpty(whiteDataBases)) {
+ Arrays.asList(whiteDataBases.trim().split(",")).forEach(whiteDataBase -> {
+ Collections.addAll(schema.dataBaseWhiteList, whiteDataBase);
+ });
}
- schema = new Schema(dataSource);
- schema.load();
- log.info("schema load successful");
- }
- private void initDataSource() throws Exception {
- Map<String, String> map = new HashMap<>();
-
- map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
- map.put("url",
- "jdbc:mysql://" + config.jdbcUrl + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
- map.put("username", config.jdbcUsername);
- map.put("password", config.jdbcPassword);
- map.put("initialSize", "2");
- map.put("maxActive", "2");
- map.put("maxWait", "60000");
- map.put("timeBetweenEvictionRunsMillis", "60000");
- map.put("minEvictableIdleTimeMillis", "300000");
- map.put("validationQuery", "SELECT 1 FROM DUAL");
- map.put("testWhileIdle", "true");
- log.info("{} config read successful", map);
- try {
- dataSource = DruidDataSourceFactory.createDataSource(map);
- } catch (Exception exception) {
- log.info("exeception,{}", exception);
- } catch (Error e) {
- log.info("error,{},e", e);
+ if (!StringUtils.isEmpty(whiteTables)) {
+ Arrays.asList(whiteTables.trim().split(",")).forEach(whiteTable -> {
+ Collections.addAll(schema.tableWhiteList, whiteTable);
+ });
}
- log.info("datasorce success");
+ schema.load();
+ log.info("load schema success");
}
}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
index 3201456..964322d 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
@@ -1,7 +1,6 @@
package org.apache.rocketmq.connect.jdbc.source;
import java.sql.Connection;
-import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -14,13 +13,12 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Queue;
import java.util.TimeZone;
-import java.util.concurrent.ConcurrentLinkedQueue;
import javax.sql.DataSource;
-import org.apache.rocketmq.connect.jdbc.Config;
+import org.apache.rocketmq.connect.jdbc.config.Config;
+import org.apache.rocketmq.connect.jdbc.common.DBUtils;
import org.apache.rocketmq.connect.jdbc.schema.Database;
import org.apache.rocketmq.connect.jdbc.schema.Schema;
import org.apache.rocketmq.connect.jdbc.schema.Table;
@@ -111,7 +109,7 @@ public class TimestampIncrementingQuerier extends Querier {
protected void createPreparedStatement(Connection conn) throws SQLException {
// Default when unspecified uses an autoincrementing column
if (incrementingColumn != null && incrementingColumn.isEmpty()) {
- incrementingColumn = JdbcUtils.getAutoincrementColumn(conn, name);
+ incrementingColumn = DBUtils.getAutoincrementColumn(conn, name);
}
String quoteString = conn.getMetaData().getIdentifierQuoteString();
@@ -205,7 +203,7 @@ public class TimestampIncrementingQuerier extends Querier {
log.info("{}·", stmt);
log.info("{},{}", incrementingOffset, timestampOffset);
return stmt.executeQuery();
- }
+ }
public List<Table> getList() {
return list;
@@ -219,10 +217,10 @@ public class TimestampIncrementingQuerier extends Querier {
try {
list.clear();
Connection conn = dataSource.getConnection();
- for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
+ for (Map.Entry<String, Database> entry : schema.getDbMap().entrySet()) {
String db = entry.getKey();
log.info("{} database is loading", db);
- Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
+ Iterator<Map.Entry<String, Table>> iterator = entry.getValue().getTableMap().entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
String tb = tableEntry.getKey();
@@ -279,20 +277,20 @@ public class TimestampIncrementingQuerier extends Querier {
} catch (Throwable exception) {
log.info("error,{}", exception);
}
- schema = new Schema(dataSource);
+ schema = new Schema(dataSource.getConnection());
schema.load();
}
- private void initDataSource() throws Exception {
+ public void initDataSource() throws Exception {
Map<String, String> map = new HashMap<>();
config = super.getConfig();
timestampColumn = config.getTimestampColmnName();
incrementingColumn = config.getIncrementingColumnName();
map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
map.put("url",
- "jdbc:mysql://" + config.jdbcUrl + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
- map.put("username", config.jdbcUsername);
- map.put("password", config.jdbcPassword);
+ "jdbc:mysql://" + config.getJdbcUrl() + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
+ map.put("username", config.getJdbcUsername());
+ map.put("password", config.getJdbcPassword());
map.put("initialSize", "2");
map.put("maxActive", "2");
map.put("maxWait", "60000");
[rocketmq-connect] 41/43: Update README.md (#553)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 5f40a886bb415d1bd84b056c41027de9666d6c65
Author: Han.G <gr...@gmail.com>
AuthorDate: Mon Feb 22 12:23:04 2021 +0100
Update README.md (#553)
The config infomation has benn verified. This update could be a draft version config template in README.md and some parts of it will probobaly be changed in the close future.
---
README.md | 83 ++++++++++++++++++++++++++++++++++++++++-----------------------
1 file changed, 53 insertions(+), 30 deletions(-)
diff --git a/README.md b/README.md
index 747e718..54c04ea 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
## rocketmq-connect-jdbc 打包
```
-mvn clean install -DskipTest -U
+mvn clean install -Dmaven.test.skip=true
```
## rocketmq-connect-jdbc 启动
@@ -11,16 +11,33 @@ mvn clean install -DskipTest -U
```
http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-source-connector-name}
-?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector",“dbUrl”:"${source-db-ip}",dbPort”:"${source-db-port}",dbUsername”:"${source-db-username}",dbPassword”:"${source-db-password}","rocketmqTopic":"jdbcTopic","mode":"bulk","whiteDataBase":{"${source-db-name}":{"${source-table-name}":{"${source-column-name}":"${source-column-value}"}}},"source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+?config={"source-rocketmq":"${runtime-ip}:${runtime-port}","source-cluster":"${broker-cluster}","connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector",“dbUrl”:"${source-db-ip}",dbPort”:"${source-db-port}",dbUsername”:"${source-db-username}",dbPassword”:"${source-db-password}","rocketmqTopic":"${source-table-name}","mode":"bulk","whiteDataBase":{"${source-db-name}":{"${source-table-name}":{"${source-column-name}":"${source-column-value}"}}},"source-record-conve [...]
+```
+
+例子
+
+```
+http://localhost:8081/connectors/jdbcConnectorSource?config={"source-rocketmq":"localhost:9876","source-cluster":"DefaultCluster",
+"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","dbUrl":"192.168.1.3","dbPort":"3306","dbUsername":"root","dbPassword":"mysqldb123456",
+"rocketmqTopic":"test_table","mode":"bulk","whiteDataBase":{"test_database":{"test_table":{"test_column":"8"}}},
+"source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
```
* **jdbc-sink-connector** 启动
```
http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-sink-connector-name}
-?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSinkConnector",“dbUrl”:"${sink-db-ip}",dbPort”:"${sink-db-port}",dbUsername”:"${sink-db-username}",dbPassword”:"${sink-db-password}","rocketmqTopic":"jdbcTopic","mode":"bulk","topicNames":"${sink-topic-name}","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+?config={"source-rocketmq":"${runtime-ip}:${runtime-port}","source-cluster":"${broker-cluster}","connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSinkConnector",“dbUrl”:"${sink-db-ip}",dbPort”:"${sink-db-port}",dbUsername”:"${sink-db-username}",dbPassword”:"${sink-db-password}","mode":"bulk","topicNames":"${source-table-name}","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
```
->**注：** `rocketmq-jdbc-connect` 的启动依赖于`rocketmq-connect-runtime`项目的启动，需将打好的`jar`包放置到`runtime`项目中`pluginPaths`配置的路径后再执行上面的启动请求,该值配置在`runtime`项目下的`connect.conf`文件中
+
+例子
+```
+http://localhost:8081/connectors/jdbcConnectorSink?config={"source-rocketmq":"localhost:9876","source-cluster":"DefaultCluster",
+"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSinkConnector","dbUrl":"192.168.1.2","dbPort":"3306","dbUsername":"root",
+"dbPassword":"mysqldb123456","topicNames":"test_table","mode":"bulk","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+```
+
+>**注：** `rocketmq-jdbc-connect` 的启动依赖于`rocketmq-connect-runtime`项目的启动，需将打好的所有`jar`包放置到`runtime`项目中`pluginPaths`配置的路径后再执行上面的启动请求,该值配置在`runtime`项目下的`connect.conf`文件中
## rocketmq-connect-jdbc 停止
@@ -31,32 +48,38 @@ http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-connector-name}/
## rocketmq-connect-jdbc 参数说明
* **jdbc-source-connector 参数说明**
-参数 | 类型 | 是否必须 | 描述 | 样例
-|---|---|---|---|---|
-|dbUrl | String | 是 | source端 DB ip | 192.168.1.2|
-|dbPort | String | 是 | source端 DB port | 3306 |
-|dbUsername | String | 是 | source端 DB 用户名 | root |
-|dbPassword | String | 是 | source端 DB 密码 | 123456 |
-|whiteDataBase | String | 是 | source端同步数据白名单，嵌套配置，为{DB名：{表名：{字段名：字段值}}}，若无指定字段数据同步，字段名可设为NO-FILTER，值为任意 | {"DATABASE_TEST":{"TEST_DATA":{"name":"test"}}} |
-|mode | String | 是 | source-connector 模式，目前仅支持bulk | bulk |
-|~~rocketmqTopic~~ | String | 是 | 待废弃 | jdbcTopic |
-|task-divide-strategy | Integer | 否 | task 分配策略, 默认值为 0，表示按照topic分配任务，每一个table便是一个topic | 0 |
-|task-parallelism | Integer | 否 | task parallelism，默认值为 1，表示将topic拆分为多少个任务进行执行 | 2 |
-|source-record-converter | String | 是 | source data 解析 | org.apache.rocketmq.connect.runtime.converter.JsonConverter |
+| KEY | TYPE | Must be filled | Description| Example
+|------------------------|---------|----------------|------------|---|
+|dbUrl | String | YES | source端 DB ip | 192.168.1.3|
+|dbPort | String | YES | source端 DB port | 3306 |
+|dbUsername | String | YES | source端 DB 用户名 | root |
+|dbPassword | String | YES | source端 DB 密码 | 123456 |
+|whiteDataBase | String | YES | source端同步数据白名单，嵌套配置，为{DB名：{表名：{字段名：字段值}}}，若无指定字段数据同步，字段名可设为NO-FILTER，值为任意 | {"DATABASE_TEST":{"TEST_DATA":{"name":"test"}}} |
+|mode | String | YES | source-connector 模式，目前仅支持bulk | bulk |
+|rocketmqTopic | String | NO | source端同步数据的topic名字，必须和要同步的数据库表名一样 | TEST_DATA |
+|task-divide-strategy | Integer | NO | task 分配策略, 默认值为 0，表示按照topic分配任务，每一个table便是一个topic | 0 |
+|task-parallelism | Integer | NO | task parallelism，默认值为 1，表示将topic拆分为多少个任务进行执行 | 2 |
+|source-rocketmq | String | YES | source 端获取路由信息连接到的RocketMQ nameserver 地址 | 192.168.1.3:9876 |
+|source-cluster | String | YES | source 端获取路由信息连接到的RocketMQ broker cluster | DefaultCluster |
+|source-record-converter | String | YES | source data 解析 | org.apache.rocketmq.connect.runtime.converter.JsonConverter |
+```
+注：1. source/sink配置文件说明是以rocketmq-connect-jdcb为demo，不同source/sink connector配置有差异，请以具体sourc/sink connector为准
+ 2. rocketmqTopic 在jdbc-source-connector中没有被用到，暂时保留的原因是为了配置显示一致性
+```
* **jdbc-sink-connector 参数说明**
-参数 | 类型 | 是否必须 | 描述 | 样例
-|---|---|---|---|---|
-|dbUrl | String | 是 | sink端 DB ip | 192.168.1.2|
-|dbPort | String | 是 | sink端 DB port | 3306 |
-|dbUsername | String | 是 | sink端 DB 用户名 | root |
-|dbPassword | String | 是 | sink端 DB 密码 | 123456 |
-|topicNames | String | 是 | sink端同步数据的topic名字 | topic-1,topic-2 |
-|mode | String | 是 | source-connector 模式，目前仅支持bulk | bulk |
-|~~rocketmqTopic~~ | String | 是 | 待废弃 | jdbcTopic |
-|task-divide-strategy | Integer | 否 | task 分配策略, 默认值为 0，表示按照topic分配任务，每一个table便是一个topic | 0 |
-|task-parallelism | Integer | 否 | task parallelism，默认值为 1，表示将topic拆分为多少个任务进行执行 | 2 |
-|source-rocketmq | String | 是 | sink 端获取路由信息连接到的RocketMQ nameserver 地址 | TODO |
-|source-rocketmq | String | 是 | sink 端获取路由信息连接到的RocketMQ broker cluster 地址 | TODO |
-|source-record-converter | String | 是 | source data 解析 | org.apache.rocketmq.connect.runtime.converter.JsonConverter |
+| KEY | TYPE | Must be filled | Description| Example
+|------------------------|---------|----------------|------------|---|
+|dbUrl | String | YES | sink端 DB ip | 192.168.1.2|
+|dbPort | String | YES | sink端 DB port | 3306 |
+|dbUsername | String | YES | sink端 DB 用户名 | root |
+|dbPassword | String | YES | sink端 DB 密码 | 123456 |
+|mode | String | YES | source-connector 模式，目前仅支持bulk | bulk |
+|topicNames | String | YES | sink端同步数据的topic名字，必须和要同步的数据库表名一样 | TEST_DATA |
+|task-divide-strategy | Integer | NO | sink端 分配策略, 默认值为 0，表示按照topic分配任务，每一个table便是一个topic | 0 |
+|task-parallelism | Integer | NO | sink端 parallelism，默认值为 1，表示将topic拆分为多少个任务进行执行 | 2 |
+|source-rocketmq | String | YES | sink端 端获取路由信息连接到的RocketMQ nameserver 地址 | 192.168.1.3:9876 |
+|source-cluster | String | YES | sink端 端获取路由信息连接到的RocketMQ broker cluster | DefaultCluster |
+|source-record-converter | String | YES | sink端 data 解析 | org.apache.rocketmq.connect.runtime.converter.JsonConverter |
+
[rocketmq-connect] 33/43: fix(jdbc-connect) removed unused class (#544)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit b9802aef6a6aced1243bda42bdf6a24764327bb9
Author: Heng Du <du...@apache.org>
AuthorDate: Mon Mar 30 12:56:18 2020 +0800
fix(jdbc-connect) removed unused class (#544)
---
.../rocketmq/connect/jdbc/config/Config.java | 31 ++++++++
.../jdbc/strategy/DivideTaskByConsistentHash.java | 82 ----------------------
.../jdbc/connector/JdbcSourceConnectorTest.java | 4 +-
3 files changed, 32 insertions(+), 85 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index ae86d3f..7b4aca3 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
@@ -78,6 +78,9 @@ public class Config {
private long timestampDelayInterval = 0;
private String dbTimezone = "GMT+8";
private String queueName;
+ private String jdbcUrl;
+ private String jdbcUsername;
+ private String jdbcPassword;
private Logger log = LoggerFactory.getLogger(Config.class);
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
@@ -322,4 +325,32 @@ public class Config {
public void setWhiteTable(String whiteTable) {
this.whiteTable = whiteTable;
}
+
+ public void setPollInterval(long pollInterval) {
+ this.pollInterval = pollInterval;
+ }
+
+ public String getJdbcUrl() {
+ return jdbcUrl;
+ }
+
+ public void setJdbcUrl(String jdbcUrl) {
+ this.jdbcUrl = jdbcUrl;
+ }
+
+ public String getJdbcUsername() {
+ return jdbcUsername;
+ }
+
+ public void setJdbcUsername(String jdbcUsername) {
+ this.jdbcUsername = jdbcUsername;
+ }
+
+ public String getJdbcPassword() {
+ return jdbcPassword;
+ }
+
+ public void setJdbcPassword(String jdbcPassword) {
+ this.jdbcPassword = jdbcPassword;
+ }
}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java
deleted file mode 100644
index bac7358..0000000
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByConsistentHash.java
+++ /dev/null
@@ -1,82 +0,0 @@
-package org.apache.rocketmq.connect.jdbc.strategy;/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import com.alibaba.fastjson.JSONObject;
-import io.openmessaging.KeyValue;
-import io.openmessaging.internal.DefaultKeyValue;
-import org.apache.rocketmq.common.consistenthash.ConsistentHashRouter;
-import org.apache.rocketmq.common.consistenthash.Node;
-import org.apache.rocketmq.replicator.config.DataType;
-import org.apache.rocketmq.replicator.config.TaskConfigEnum;
-import org.apache.rocketmq.replicator.config.TaskDivideConfig;
-import org.apache.rocketmq.replicator.config.TaskTopicInfo;
-
-import java.util.*;
-
-public class DivideTaskByConsistentHash extends TaskDivideStrategy {
- @Override public List<KeyValue> divide(Map<String, List<TaskTopicInfo>> topicMap, TaskDivideConfig tdc) {
-
- List<KeyValue> config = new ArrayList<>();
- int parallelism = tdc.getTaskParallelism();
- Map<Integer, List<TaskTopicInfo>> queueTopicList = new HashMap<>();
- int id = -1;
-
- Collection<ClientNode> cidNodes = new ArrayList<>();
- for (int i = 0; i < parallelism; i++) {
- cidNodes.add(new ClientNode(i, Integer.toString(i)));
- queueTopicList.put(i, new ArrayList<>());
- }
-
- ConsistentHashRouter<ClientNode> router = new ConsistentHashRouter<>(cidNodes, cidNodes.size());
-
- for (String t : topicMap.keySet()) {
- for (TaskTopicInfo queue : topicMap.get(t)) {
- ClientNode clientNode = router.routeNode(queue.toString());
- if (clientNode != null) {
- queueTopicList.get(clientNode.index).add(queue);
- }
- }
- }
-
- for (int i = 0; i < parallelism; i++) {
- KeyValue keyValue = new DefaultKeyValue();
- keyValue.put(TaskConfigEnum.TASK_STORE_ROCKETMQ.getKey(), tdc.getStoreTopic());
- keyValue.put(TaskConfigEnum.TASK_SOURCE_ROCKETMQ.getKey(), tdc.getSourceNamesrvAddr());
- keyValue.put(TaskConfigEnum.TASK_DATA_TYPE.getKey(), DataType.COMMON_MESSAGE.ordinal());
- keyValue.put(TaskConfigEnum.TASK_TOPIC_INFO.getKey(), JSONObject.toJSONString(queueTopicList.get(i)));
- keyValue.put(TaskConfigEnum.TASK_SOURCE_RECORD_CONVERTER.getKey(), tdc.getSrcRecordConverter());
- config.add(keyValue);
- }
-
- return config;
- }
-
- private static class ClientNode implements Node {
- private final String clientID;
- private final int index;
-
- public ClientNode(int index, String clientID) {
- this.index = index;
- this.clientID = clientID;
- }
-
- @Override
- public String getKey() {
- return clientID;
- }
- }
-}
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
index 1e7cf78..5d25f98 100644
--- a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
@@ -22,9 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.util.HashSet;
import java.util.Set;
-import org.apache.rocketmq.connect.jdbc.Config;
-import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
-
+import org.apache.rocketmq.connect.jdbc.config.Config;
import org.junit.Test;
import io.openmessaging.KeyValue;
[rocketmq-connect] 35/43: 1.add required fields in RockeMQ jdbc connector 2. add docs (#539)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 5420e44a476b726faf88d4c313dc10e11574702d
Author: affe <af...@gmail.com>
AuthorDate: Sun Mar 29 22:08:12 2020 -0700
1.add required fields in RockeMQ jdbc connector 2. add docs (#539)
---
README.md | 2 ++
.../java/org/apache/rocketmq/connect/jdbc/config/Config.java | 12 +++++++-----
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
index 7b3d397..747e718 100644
--- a/README.md
+++ b/README.md
@@ -57,4 +57,6 @@ http://${runtime-ip}:${runtime-port}/connectors/${rocketmq-jdbc-connector-name}/
|~~rocketmqTopic~~ | String | 是 | 待废弃 | jdbcTopic |
|task-divide-strategy | Integer | 否 | task 分配策略, 默认值为 0，表示按照topic分配任务，每一个table便是一个topic | 0 |
|task-parallelism | Integer | 否 | task parallelism，默认值为 1，表示将topic拆分为多少个任务进行执行 | 2 |
+|source-rocketmq | String | 是 | sink 端获取路由信息连接到的RocketMQ nameserver 地址 | TODO |
+|source-rocketmq | String | 是 | sink 端获取路由信息连接到的RocketMQ broker cluster 地址 | TODO |
|source-record-converter | String | 是 | source data 解析 | org.apache.rocketmq.connect.runtime.converter.JsonConverter |
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index 7b4aca3..cca1aa5 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
@@ -85,12 +85,14 @@ public class Config {
private Logger log = LoggerFactory.getLogger(Config.class);
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
{
- add("dbUrl");
- add("dbPort");
- add("dbUsername");
- add("dbPassword");
- add("mode");
+ add(CONN_DB_IP);
+ add(CONN_DB_PORT);
+ add(CONN_DB_USERNAME);
+ add(CONN_DB_PASSWORD);
+ add(CONN_DB_MODE);
add("rocketmqTopic");
+ add(CONN_SOURCE_RMQ);
+ add(CONN_SOURCE_CLUSTER);
}
};
[rocketmq-connect] 15/43: Update Querier.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit d0569b7e91ee784d53ef6e4f723a1ab0f02453d8
Author: Yuchen Li <yu...@126.com>
AuthorDate: Fri Aug 9 13:49:37 2019 +0800
Update Querier.java
---
src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index 3d5b2c6..073a896 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -120,8 +120,6 @@ public class Querier {
for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
String db = entry.getKey();
- if (!db.contains("jdbc_db"))
- continue;
Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
[rocketmq-connect] 37/43: [ISSUE #550] Removed unnecessary value of REQUEST_CONFIG in Config.java (#551)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit a1f2ff2d888878e07f2256770d49ccb912ff81bf
Author: Han.G <gr...@gmail.com>
AuthorDate: Tue Apr 7 03:45:10 2020 +0200
[ISSUE #550] Removed unnecessary value of REQUEST_CONFIG in Config.java (#551)
---
src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index 9162bad..fe5a90b 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
@@ -87,7 +87,6 @@ public class Config {
add(CONN_DB_USERNAME);
add(CONN_DB_PASSWORD);
add(CONN_DB_MODE);
- add("rocketmqTopic");
add(CONN_SOURCE_RMQ);
add(CONN_SOURCE_CLUSTER);
}
[rocketmq-connect] 02/43: Add Jdbc Source Connector and Do Unit Test
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 13b6ae3ba93a56b156f872dbac4914faf34d9050
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Thu Jul 18 22:52:55 2019 +0800
Add Jdbc Source Connector and Do Unit Test
---
pom.xml | 195 +++++++++++++++++++++
.../org/apache/rocketmq/connect/jdbc/Config.java | 137 +++++++++++++++
.../jdbc/connector/JdbcSourceConnector.java | 83 +++++++++
.../jdbc/connector/JdbcSourceConnectorTest.java | 85 +++++++++
4 files changed, 500 insertions(+)
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..361dd77
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,195 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-connect-jdbc</artifactId>
+ <version>1.0.0</version>
+
+ <name>connect-jdbc</name>
+ <url>https://github.com/apache/incubator-rocketmq-externals/tree/master/rocketmq-connect-jdbc</url>
+
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ </license>
+ </licenses>
+
+ <issueManagement>
+ <system>jira</system>
+ <url>https://issues.apache.org/jira/browse/RocketMQ</url>
+ </issueManagement>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+
+ <!-- Compiler settings properties -->
+ <maven.compiler.source>1.8</maven.compiler.source>
+ <maven.compiler.target>1.8</maven.compiler.target>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>versions-maven-plugin</artifactId>
+ <version>2.3</version>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>clirr-maven-plugin</artifactId>
+ <version>2.7</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.6.1</version>
+ <configuration>
+ <source>${maven.compiler.source}</source>
+ <target>${maven.compiler.target}</target>
+ <compilerVersion>${maven.compiler.source}</compilerVersion>
+ <showDeprecation>true</showDeprecation>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.19.1</version>
+ <configuration>
+ <argLine>-Xms512m -Xmx1024m</argLine>
+ <forkMode>always</forkMode>
+ <includes>
+ <include>**/*Test.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.6</version>
+ <configuration>
+ <locales>en_US</locales>
+ <outputEncoding>UTF-8</outputEncoding>
+ <inputEncoding>UTF-8</inputEncoding>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>3.0.1</version>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.10.4</version>
+ <configuration>
+ <charset>UTF-8</charset>
+ <locale>en_US</locale>
+ <excludePackageNames>io.openmessaging.internal</excludePackageNames>
+ </configuration>
+ <executions>
+ <execution>
+ <id>aggregate</id>
+ <goals>
+ <goal>aggregate</goal>
+ </goals>
+ <phase>site</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>3.0.2</version>
+ <configuration>
+ <encoding>${project.build.sourceEncoding}</encoding>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <version>3.0.4</version>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.assertj</groupId>
+ <artifactId>assertj-core</artifactId>
+ <version>2.6.0</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>2.6.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.openmessaging</groupId>
+ <artifactId>openmessaging-connector</artifactId>
+ <version>0.1.0-beta</version>
+ </dependency>
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>fastjson</artifactId>
+ <version>1.2.51</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.7</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.0.13</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>1.0.13</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.rocketmq</groupId>
+ <artifactId>rocketmq-openmessaging</artifactId>
+ <version>4.3.2</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>javax.jms</groupId>
+ <artifactId>javax.jms-api</artifactId>
+ <version>2.0</version>
+ </dependency>
+
+ </dependencies>
+
+</project>
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
new file mode 100644
index 0000000..3bc3032
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc;
+
+import io.openmessaging.KeyValue;
+import java.lang.reflect.Method;
+import java.util.HashSet;
+import java.util.Set;
+import javax.jms.Session;
+
+public class Config {
+ @SuppressWarnings("serial")
+
+ public String jdbcAddr;
+ public Integer jdbcPort;
+ public String jdbcUsername;
+ public String jdbcPassword;
+
+ public String queueName;
+
+ public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
+ {
+ add("jdbcAddr");
+ add("jdbcPort");
+ add("jdbcUsername");
+ add("jdbcPassword");
+ }
+ };
+
+// public String destinationType;
+//
+// public String destinationName;
+//
+// public String messageSelector;
+//
+// private Integer sessionAcknowledgeMode = Session.AUTO_ACKNOWLEDGE;
+//
+// private Boolean sessionTransacted = Boolean.FALSE;
+
+ public void load(KeyValue props) {
+
+ properties2Object(props, this);
+ }
+
+ private void properties2Object(final KeyValue p, final Object object) {
+ //Java Reflection Application
+ Method[] methods = object.getClass().getMethods();
+ for (Method method : methods) {
+ String mn = method.getName();
+ if (mn.startsWith("set")) {
+ try {
+ String tmp = mn.substring(4);
+ String first = mn.substring(3, 4);
+
+ String key = first.toLowerCase() + tmp;
+ String property = p.getString(key);
+ if (property != null) {
+ Class<?>[] pt = method.getParameterTypes();
+ if (pt != null && pt.length > 0) {
+ String cn = pt[0].getSimpleName();
+ Object arg = null;
+ if (cn.equals("int") || cn.equals("Integer")) {
+ arg = Integer.parseInt(property);
+ } else if (cn.equals("long") || cn.equals("Long")) {
+ arg = Long.parseLong(property);
+ } else if (cn.equals("double") || cn.equals("Double")) {
+ arg = Double.parseDouble(property);
+ } else if (cn.equals("boolean") || cn.equals("Boolean")) {
+ arg = Boolean.parseBoolean(property);
+ } else if (cn.equals("float") || cn.equals("Float")) {
+ arg = Float.parseFloat(property);
+ } else if (cn.equals("String")) {
+ arg = property;
+ } else {
+ continue;
+ }
+ method.invoke(object, arg);
+
+ }
+ }
+ } catch (Throwable ignored) {
+ }
+ }
+ }
+ }
+
+ public String getJdbcAddr() {
+ return jdbcAddr;
+ }
+
+ public void setJdbcAddr(String JdbcAddr) {
+ this.jdbcAddr = jdbcAddr;
+ }
+
+ public int getJdbcPort() {
+ return jdbcPort;
+ }
+
+ public void setJdbcPort(Integer jdbcPort) {
+ this.jdbcPort = jdbcPort;
+ }
+
+ public String getJdbcUsername() {
+ return jdbcUsername;
+ }
+
+ public void setJdbcUsername(String jdbcUsername) {
+ this.jdbcUsername = jdbcUsername;
+ }
+
+ public String getJdbcPassword() {
+ return jdbcPassword;
+ }
+
+ public void setJdbcPassword(String jdbcPassword) {
+ this.jdbcPassword = jdbcPassword;
+ }
+
+
+
+
+}
\ No newline at end of file
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
new file mode 100644
index 0000000..d6b0e3b
--- /dev/null
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.connector;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import io.openmessaging.connector.api.source.SourceConnector;
+
+import org.apache.rocketmq.connect.jdbc.Config;
+//import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JdbcSourceConnector extends SourceConnector {
+ private static final Logger log = LoggerFactory.getLogger(JdbcSourceConnector.class);
+ private KeyValue config;
+
+ @Override
+ public String verifyAndSetConfig(KeyValue config) {
+ System.out.println(11+config.toString());
+ log.info("JdbcSourceConnector verifyAndSetConfig enter");
+ for (String requestKey : Config.REQUEST_CONFIG) {
+ System.out.println(12+requestKey);
+ if (!config.containsKey(requestKey)) {
+ return "Request config key: " + requestKey;
+ }
+ }
+ this.config = config;
+ System.out.println(11+config.toString());
+ return "";
+ }
+
+ @Override
+ public void start() {
+
+ }
+
+ @Override
+ public void stop() {
+
+ }
+
+ @Override public void pause() {
+
+ }
+
+ @Override public void resume() {
+
+ }
+
+ @Override
+ public Class<? extends Task> taskClass(){
+ return JdbcSourceTask.class;
+ }
+
+ @Override
+ public List<KeyValue> taskConfigs() {
+ List<KeyValue> config = new ArrayList<>();
+ config.add(this.config);
+ return config;
+ }
+
+ // abstract Set<String> getRequiredConfig();
+}
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
new file mode 100644
index 0000000..79e4e59
--- /dev/null
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.connector;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.rocketmq.connect.jdbc.Config;
+import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
+
+import org.junit.Test;
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.Task;
+import io.openmessaging.internal.DefaultKeyValue;
+
+public class JdbcSourceConnectorTest {
+
+ public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
+ {
+ add("jdbcAddr");
+ add("jdbcPort");
+ add("jdbcUsername");
+ add("jdbcPassword");
+ }
+ };
+
+ JdbcSourceConnector connector = new JdbcSourceConnector() {
+
+
+ @Override
+ public Class<? extends Task> taskClass() {
+ return JdbcSourceTask.class;
+ }
+
+
+// Set<String> getRequiredConfig() {
+// return REQUEST_CONFIG;
+// }
+ };
+
+ @Test
+ public void verifyAndSetConfigTest() {
+ KeyValue keyValue = new DefaultKeyValue();
+
+ for (String requestKey : Config.REQUEST_CONFIG) {
+ assertEquals(connector.verifyAndSetConfig(keyValue), "Request config key: " + requestKey);
+ keyValue.put(requestKey, requestKey);
+ }
+ assertEquals(connector.verifyAndSetConfig(keyValue), "");
+ }
+
+ @Test
+ public void taskClassTest() {
+ assertEquals(connector.taskClass(), JdbcSourceTask.class);
+ }
+
+ @Test
+ public void taskConfigsTest() {
+ assertEquals(connector.taskConfigs().get(0), null);
+ KeyValue keyValue = new DefaultKeyValue();
+ for (String requestKey : Config.REQUEST_CONFIG) {
+ keyValue.put(requestKey, requestKey);
+ }
+ connector.verifyAndSetConfig(keyValue);
+ assertEquals(connector.taskConfigs().get(0), keyValue);
+ }
+}
[rocketmq-connect] 07/43: Add Config File
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 2cc48d9a56f38abc4a82296e4e6e3d8b3c190813
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Tue Jul 23 22:43:01 2019 +0800
Add Config File
---
.../org/apache/rocketmq/connect/jdbc/Config.java | 224 +++++++++++++++++++--
1 file changed, 208 insertions(+), 16 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
index 2b485e4..217b449 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
@@ -18,27 +18,56 @@
package org.apache.rocketmq.connect.jdbc;
import io.openmessaging.KeyValue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.lang.reflect.Method;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
-import javax.jms.Session;
-
public class Config {
@SuppressWarnings("serial")
- public String jdbcAddr;
- public Integer jdbcPort;
+ private static final Logger LOG = LoggerFactory.getLogger(Config.class);
+
+ /* Database Connection Config */
+ public String jdbcUrl;
public String jdbcUsername;
public String jdbcPassword;
+ public String rocketmqTopic;
+ public String jdbcBackoff;
+ public String jdbcAttempts;
+ public String catalogPattern=null;
+ public List tableWhitelist;
+ public List tableBlacklist;
+ public String schemaPattern=null;
+ public boolean numericPrecisionMapping=false;
+ public String bumericMapping=null;
+ public String dialectName="";
+ /* Mode Config */
+ public String mode="";
+ public String incrementingColumnName= "";
+ public String query="";
+ public String timestampColmnName="";
+ public boolean validateNonNull=true;
+
+ /*Connector config*/
+ public String tableTypes="table";
+ public int pollInterval=5000;
+ public int batchMaxRows=100;
+ public long tablePollInterval=60000;
+ public long timestampDelayInterval=0;
+ public String dbTimezone="UTC";
public String queueName;
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
{
- add("jdbcAddr");
- add("jdbcPort");
+ add("jdbcUrl");
add("jdbcUsername");
add("jdbcPassword");
+ add("mode");
+ add("queueName");
}
};
@@ -49,7 +78,6 @@ public class Config {
}
private void properties2Object(final KeyValue p, final Object object) {
- //Java Reflection Application
Method[] methods = object.getClass().getMethods();
for (Method method : methods) {
String mn = method.getName();
@@ -90,20 +118,20 @@ public class Config {
}
}
- public String getJdbcAddr() {
- return jdbcAddr;
+ public String getQueueName() {
+ return queueName;
}
- public void setJdbcAddr(String JdbcAddr) {
- this.jdbcAddr = jdbcAddr;
+ public void setQueueName(String queueName) {
+ this.queueName = queueName;
}
- public int getJdbcPort() {
- return jdbcPort;
+ public String getJdbcUrl() {
+ return jdbcUrl;
}
- public void setJdbcPort(Integer jdbcPort) {
- this.jdbcPort = jdbcPort;
+ public void setJdbcUrl(String jdbcUrl) {
+ this.jdbcUrl = jdbcUrl;
}
public String getJdbcUsername() {
@@ -122,7 +150,171 @@ public class Config {
this.jdbcPassword = jdbcPassword;
}
+ public String getRocketmqTopic() {
+ return rocketmqTopic;
+ }
+
+ public void setRocketmqTopic(String rocketmqTopic) {
+ this.rocketmqTopic = rocketmqTopic;
+ }
+
+ public String getJdbcBackoff() {
+ return jdbcBackoff;
+ }
+
+ public void setJdbcBackoff(String jdbcBackoff) {
+ this.jdbcBackoff = jdbcBackoff;
+ }
+
+ public String getJdbcAttempts() {
+ return jdbcAttempts;
+ }
+
+ public void setJdbcAttempts(String jdbcAttempts) {
+ this.jdbcAttempts = jdbcAttempts;
+ }
+
+ public String getCatalogPattern() {
+ return catalogPattern;
+ }
+
+ public void setCatalogPattern(String catalogPattern) {
+ this.catalogPattern = catalogPattern;
+ }
+
+ public List getTableWhitelist() {
+ return tableWhitelist;
+ }
+
+ public void setTableWhitelist(List tableWhitelist) {
+ this.tableWhitelist = tableWhitelist;
+ }
+
+ public List getTableBlacklist() {
+ return tableBlacklist;
+ }
+
+ public void setTableBlacklist(List tableBlacklist) {
+ this.tableBlacklist = tableBlacklist;
+ }
+
+ public String getSchemaPattern() {
+ return schemaPattern;
+ }
+
+ public void setSchemaPattern(String schemaPattern) {
+ this.schemaPattern = schemaPattern;
+ }
+
+ public boolean isNumericPrecisionMapping() {
+ return numericPrecisionMapping;
+ }
+
+ public void setNumericPrecisionMapping(boolean numericPrecisionMapping) {
+ this.numericPrecisionMapping = numericPrecisionMapping;
+ }
+
+ public String getBumericMapping() {
+ return bumericMapping;
+ }
+
+ public void setBumericMapping(String bumericMapping) {
+ this.bumericMapping = bumericMapping;
+ }
+
+ public String getDialectName() {
+ return dialectName;
+ }
+
+ public void setDialectName(String dialectName) {
+ this.dialectName = dialectName;
+ }
+
+ public String getMode() {
+ return mode;
+ }
+
+ public void setMode(String mode) {
+ this.mode = mode;
+ }
+
+ public String getIncrementingColumnName() {
+ return incrementingColumnName;
+ }
+
+ public void setIncrementingColumnName(String incrementingColumnName) {
+ this.incrementingColumnName = incrementingColumnName;
+ }
+
+ public String getQuery() {
+ return query;
+ }
+
+ public void setQuery(String query) {
+ this.query = query;
+ }
+
+ public String getTimestampColmnName() {
+ return timestampColmnName;
+ }
+
+ public void setTimestampColmnName(String timestampColmnName) {
+ this.timestampColmnName = timestampColmnName;
+ }
+
+ public boolean isValidateNonNull() {
+ return validateNonNull;
+ }
+
+ public void setValidateNonNull(boolean validateNonNull) {
+ this.validateNonNull = validateNonNull;
+ }
+ public String getTableTypes() {
+ return tableTypes;
+ }
+
+ public void setTableTypes(String tableTypes) {
+ this.tableTypes = tableTypes;
+ }
+ public int getPollInterval() {
+ return pollInterval;
+ }
+
+ public void setPollInterval(int pollInterval) {
+ this.pollInterval = pollInterval;
+ }
-}
+ public int getBatchMaxRows() {
+ return batchMaxRows;
+ }
+
+ public void setBatchMaxRows(int batchMaxRows) {
+ this.batchMaxRows = batchMaxRows;
+ }
+
+ public long getTablePollInterval() {
+ return tablePollInterval;
+ }
+
+ public void setTablePollInterval(long tablePollInterval) {
+ this.tablePollInterval = tablePollInterval;
+ }
+
+ public long getTimestampDelayInterval() {
+ return timestampDelayInterval;
+ }
+
+ public void setTimestampDelayInterval(long timestampDelayInterval) {
+ this.timestampDelayInterval = timestampDelayInterval;
+ }
+
+ public String getDbTimezone() {
+ return dbTimezone;
+ }
+
+ public void setDbTimezone(String dbTimezone) {
+ this.dbTimezone = dbTimezone;
+ }
+}
\ No newline at end of file
[rocketmq-connect] 38/43: [ISSUE #554] Update druid version and set ConnectionErrorRetryAttempts
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 9849db65544037a2b882300b96305f868e82be7e
Author: affe <af...@gmail.com>
AuthorDate: Mon Jul 6 10:04:03 2020 +0800
[ISSUE #554] Update druid version and set ConnectionErrorRetryAttempts
---
pom.xml | 2 +-
.../rocketmq/connect/jdbc/common/DBUtils.java | 31 +++++++++++-----------
2 files changed, 17 insertions(+), 16 deletions(-)
diff --git a/pom.xml b/pom.xml
index 61680f1..2feaa22 100644
--- a/pom.xml
+++ b/pom.xml
@@ -263,7 +263,7 @@
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
- <version>1.0.31</version>
+ <version>1.1.22</version>
</dependency>
</dependencies>
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java b/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
index 31a86d1..963fbf6 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
@@ -17,6 +17,7 @@
package org.apache.rocketmq.connect.jdbc.common;
+import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidDataSourceFactory;
import org.apache.rocketmq.connect.jdbc.config.Config;
import org.apache.rocketmq.connect.jdbc.connector.JdbcSourceTask;
@@ -189,21 +190,21 @@ public class DBUtils {
}
public static DataSource initDataSource(Config config) throws Exception {
- Map<String, String> map = new HashMap<>();
- map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
- map.put("url",
- "jdbc:mysql://" + config.getDbUrl() + ":" + config.getDbPort() + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8&characterEncoding=utf8");
- map.put("username", config.getDbUsername());
- map.put("password", config.getDbPassword());
- map.put("initialSize", "1");
- map.put("maxActive", "2");
- map.put("maxWait", "60000");
- map.put("timeBetweenEvictionRunsMillis", "60000");
- map.put("minEvictableIdleTimeMillis", "300000");
- map.put("validationQuery", "SELECT 1 FROM DUAL");
- map.put("testWhileIdle", "true");
- log.info("{} config read successful", map);
- DataSource dataSource = DruidDataSourceFactory.createDataSource(map);
+ DruidDataSource dataSource = new DruidDataSource();
+ dataSource.setDriverClassName("com.mysql.cj.jdbc.Driver");
+ dataSource.setUrl("jdbc:mysql://" + config.getDbUrl() + ":" + config.getDbPort() + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8&characterEncoding=utf8");
+ dataSource.setUsername(config.getDbUsername());
+ dataSource.setPassword(config.getDbPassword());
+ dataSource.setInitialSize(1);
+ dataSource.setMaxActive(2);
+ dataSource.setMaxWait(60000);
+ dataSource.setTimeBetweenEvictionRunsMillis(60000);
+ dataSource.setConnectionErrorRetryAttempts(2);
+ dataSource.setBreakAfterAcquireFailure(true);
+ dataSource.setMinEvictableIdleTimeMillis(300000);
+ dataSource.setValidationQuery("SELECT 1 FROM DUAL");
+ dataSource.setTestWhileIdle(true);
+
log.info("init data source success");
return dataSource;
}
[rocketmq-connect] 09/43: Add JdbcSourceTask and Schema
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit e915f678c601912fec4b1f7438972f740ca5288b
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Mon Jul 29 21:49:58 2019 +0800
Add JdbcSourceTask and Schema
---
pom.xml | 12 +++
.../apache/rocketmq/connect/jdbc/Replicator.java | 118 ---------------------
.../rocketmq/connect/jdbc/source/Querier.java | 2 -
.../jdbc/connector/JdbcSourceConnectorTest.java | 9 +-
.../connect/jdbc/connector/JdbcSourceTaskTest.java | 44 ++++++++
5 files changed, 60 insertions(+), 125 deletions(-)
diff --git a/pom.xml b/pom.xml
index a53b8b0..dee7710 100644
--- a/pom.xml
+++ b/pom.xml
@@ -150,6 +150,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.12</version>
+ </dependency>
+ <dependency>
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-connector</artifactId>
<version>0.1.0-beta</version>
@@ -179,11 +184,18 @@
<artifactId>rocketmq-openmessaging</artifactId>
<version>4.3.2</version>
</dependency>
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>druid</artifactId>
+ <version>1.0.18</version>
+ </dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
</dependency>
+
+
</dependencies>
</project>
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Replicator.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Replicator.java
deleted file mode 100644
index b24b7e5..0000000
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/Replicator.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.connect.jdbc;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class Replicator {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(Replicator.class);
-
- private static final Logger POSITION_LOGGER = LoggerFactory.getLogger("PositionLogger");
-
- private Config config;
-
- private EventProcessor eventProcessor;
-
- private Object lock = new Object();
- private BinlogPosition nextBinlogPosition;
- private long nextQueueOffset;
- private long xid;
- private BlockingQueue<Transaction> queue = new LinkedBlockingQueue<>();
-
- public Replicator(Config config){
- this.config = config;
- }
-
- public void start() {
-
- try {
-
- eventProcessor = new EventProcessor(this);
- eventProcessor.start();
-
- } catch (Exception e) {
- LOGGER.error("Start error.", e);
- }
- }
-
- public void stop(){
- eventProcessor.stop();
- }
-
- public void commit(Transaction transaction, boolean isComplete) {
-
- queue.add(transaction);
- for (int i = 0; i < 3; i++) {
- try {
- if (isComplete) {
- long offset = 1;
- synchronized (lock) {
- xid = transaction.getXid();
- nextBinlogPosition = transaction.getNextBinlogPosition();
- nextQueueOffset = offset;
- }
-
- } else {
- }
- break;
-
- } catch (Exception e) {
- LOGGER.error("Push error,retry:" + (i + 1) + ",", e);
- }
- }
- }
-
- public void logPosition() {
-
- String binlogFilename = null;
- long xid = 0L;
- long nextPosition = 0L;
- long nextOffset = 0L;
-
- synchronized (lock) {
- if (nextBinlogPosition != null) {
- xid = this.xid;
- binlogFilename = nextBinlogPosition.getBinlogFilename();
- nextPosition = nextBinlogPosition.getPosition();
- nextOffset = nextQueueOffset;
- }
- }
-
- if (binlogFilename != null) {
- POSITION_LOGGER.info("XID: {}, BINLOG_FILE: {}, NEXT_POSITION: {}, NEXT_OFFSET: {}",
- xid, binlogFilename, nextPosition, nextOffset);
- }
-
- }
-
- public Config getConfig() {
- return config;
- }
-
-// public BinlogPosition getNextBinlogPosition() {
-// return nextBinlogPosition;
-// }
-
- public BlockingQueue<Transaction> getQueue() {
- return queue;
- }
-}
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index 61323d4..e99da74 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
@@ -125,8 +125,6 @@ public class Querier {
for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
String db = entry.getKey();
- if(!db.contains("jdbc_db"))
- continue;
Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Table> tableEntry = iterator.next();
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
index 79e4e59..97d87ee 100644
--- a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
@@ -35,10 +35,11 @@ public class JdbcSourceConnectorTest {
public static final Set<String> REQUEST_CONFIG = new HashSet<String>() {
{
- add("jdbcAddr");
- add("jdbcPort");
+ add("jdbcUrl");
add("jdbcUsername");
add("jdbcPassword");
+ add("mode");
+ add("rocketmqTopic");
}
};
@@ -51,9 +52,7 @@ public class JdbcSourceConnectorTest {
}
-// Set<String> getRequiredConfig() {
-// return REQUEST_CONFIG;
-// }
+
};
@Test
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java
new file mode 100644
index 0000000..f9c8c6f
--- /dev/null
+++ b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.rocketmq.connect.jdbc.connector;
+import java.util.Collection;
+import org.junit.Test;
+
+
+import io.openmessaging.KeyValue;
+import io.openmessaging.connector.api.data.SourceDataEntry;
+import io.openmessaging.internal.DefaultKeyValue;
+
+public class JdbcSourceTaskTest {
+
+
+ @Test
+ public void test() throws InterruptedException {
+ KeyValue kv = new DefaultKeyValue();
+ kv.put("jdbcUrl","localhost:3306");
+ kv.put("jdbcUsername","root");
+ kv.put("jdbcPassword","199812160");
+ kv.put("mode","bulk");
+ kv.put("rocketmqTopic","JdbcTopic");
+ JdbcSourceTask task = new JdbcSourceTask();
+ task.start(kv);
+ Collection<SourceDataEntry> sourceDataEntry = task.poll();
+ System.out.println(sourceDataEntry);
+
+ }
+}
[rocketmq-connect] 21/43: update readme
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 8a266e862ffc8ffd96586a4e7e4c43c7080d8390
Author: yuchenlichuck <yu...@126.com>
AuthorDate: Sat Aug 17 10:37:32 2019 +0800
update readme
---
README.md | 48 ++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 40 insertions(+), 8 deletions(-)
diff --git a/README.md b/README.md
index 32da4b5..1970378 100644
--- a/README.md
+++ b/README.md
@@ -127,11 +127,7 @@ storePathRootDir=/home/connect/storeRoot
httpPort=8081
```
-
-
-
-
-- b、日志相关配置在logback.xml中修改
+b、日志相关配置在logback.xml中修改
```
注：rocketmq需要先创建cluster-topic，config-topic，offset-topic，position-topic
@@ -179,9 +175,9 @@ java -cp .;./conf/;./lib/* org.apache.rocketmq.connect.runtime.ConnectStartup -c
mvn dependency:copy-dependencies
```
+#### Bulk查询方法，在http中输入Get 请求（目前仅适配过MYSQL）
-
-已实现Bulk查询方法，在http中输入Get 请求（目前仅适配过MYSQL）
+请将jdbcUrl, jdbcUsername, jdbcPassword，改为所连接数据库的配置
```http
http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"bulk","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
@@ -198,6 +194,42 @@ http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":
2019-08-09 11:33:27 INFO pool-9-thread-1 - schema load successful
2019-08-09 11:33:27 INFO pool-9-thread-1 - querier.poll
+#### Incrementing and / or Timestamp Querier
+
+- 要使用的时间戳和/或自增列必须在连接器处理的所有表上。如果不同的表具有不同名称的时间戳/自增列，则需要创建单独的连接器配置；
+- 可以结合使用这些方法中的（时间戳/自增）或两者（时间戳+自增）；
+
+##### Incrementing Querier （自增列查询）
+
+注：表中需要含递增的一列，如果只使用自增列，则不会捕获对数据的更新，除非每次更新时自增列也会增加。
+
+```http
+http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"incrementing","incrementingColumnName":"id,"source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+```
+
+##### Timestamp Querier （时间戳查询）
+
+
+```http
+http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"timestamp":"timestampColumnName","id","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"}
+```
+
+##### Incrementing + Timestamp Querier (自增列+时间戳查询)
+
+
+```http
+http://127.0.0.1:8081/connectors/testSourceConnector1?config={"connector-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","jdbcUrl":"127.0.0.1:3306","jdbcUsername":"root","jdbcPassword":"123456","task-class":"org.apache.rocketmq.connect.jdbc.connector.JdbcSourceConnector","rocketmqTopic":"jdbcTopic","mode":"incrementing+timestamp","timestampColumnName":"timestamp","incrementingColumnName":"id","source-record-converter":"org.apache.rocketmq.connect.runtime.converter [...]
+```
+
+日志如果显示为如下，则connect成功。
+
+2019-08-15 14:09:43 INFO RebalanceService - JdbcSourceConnector verifyAndSetConfig enter
+2019-08-15 14:09:44 INFO pool-14-thread-1 - Config.load.start
+2019-08-15 14:09:44 INFO pool-14-thread-1 - config load successfully
+2019-08-15 14:09:44 INFO pool-14-thread-1 - map start,199812160
+2019-08-15 14:09:44 INFO pool-14-thread-1 - {password=199812160, validationQuery=SELECT 1 FROM DUAL, testWhileIdle=true, timeBetweenEvictionRunsMillis=60000, minEvictableIdleTimeMillis=300000, initialSize=2, driverClassName=com.mysql.cj.jdbc.Driver, maxWait=60000, url=jdbc:mysql://127.0.0.1:3306?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8, username=root, maxActive=2}config read successfully
+
3、启动sinkConnector
-To Be Continued.
\ No newline at end of file
+To Be Continued.
+
[rocketmq-connect] 40/43: [ISSUE #570] ASoC connect runtime optimization: CLI (#622)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit b425260296d1524d61643b3e286d395df1ed5835
Author: Dreaouth <jl...@163.com>
AuthorDate: Sun Sep 20 19:09:29 2020 +0800
[ISSUE #570] ASoC connect runtime optimization: CLI (#622)
feature(rocketmq-runtime) add CLI support for rocketmq-connect-runtime
* Add CLI
* Fix checkstyle
* Optimize CLI structure
* Add README.md
* Rename CLI
* Update pom.xml
* Optimize the connectors and tasks format
* Fix newline format
---
.../connect/jdbc/connector/JdbcSourceConnector.java | 2 +-
.../connect/jdbc/strategy/DivideTaskByTopic.java | 17 ++++++++++-------
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
index a083e84..ee62133 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
@@ -59,7 +59,7 @@ public class JdbcSourceConnector extends SourceConnector {
@Override
public void start() {
-
+ log.info("JdbcSourceConnector start");
}
@Override
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
index c1d5020..5762795 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
@@ -41,21 +41,23 @@ public class DivideTaskByTopic extends TaskDivideStrategy {
int parallelism = tdc.getTaskParallelism();
int id = -1;
Map<String, String> topicRouteMap = ((SourceDbConnectorConfig)dbConnectorConfig).getWhiteTopics();
- Map<Integer, Map<String, Map<String, String>>> taskTopicList = new HashMap<>();
+ Map<Integer, String> taskTopicList = new HashMap<>();
+ Map<Integer, Map<String, Map<String, String>>> taskWhiteList = new HashMap<>();
for (Map.Entry<String, String> entry : topicRouteMap.entrySet()) {
int ind = ++id % parallelism;
- if (!taskTopicList.containsKey(ind)) {
- taskTopicList.put(ind, new HashMap<>());
+ if (!taskWhiteList.containsKey(ind)) {
+ taskWhiteList.put(ind, new HashMap<>());
}
String dbKey = entry.getKey().split("-")[0];
String tableKey = entry.getKey().split("-")[1];
+ taskTopicList.put(ind, tableKey);
String filter = entry.getValue();
Map<String, String> tableMap = new HashMap<>();
tableMap.put(tableKey, filter);
- if(!taskTopicList.get(ind).containsKey(dbKey)){
- taskTopicList.get(ind).put(dbKey, tableMap);
+ if(!taskWhiteList.get(ind).containsKey(dbKey)){
+ taskWhiteList.get(ind).put(dbKey, tableMap);
}else {
- taskTopicList.get(ind).get(dbKey).putAll(tableMap);
+ taskWhiteList.get(ind).get(dbKey).putAll(tableMap);
}
}
@@ -66,7 +68,8 @@ public class DivideTaskByTopic extends TaskDivideStrategy {
keyValue.put(Config.CONN_DB_PORT, tdc.getDbPort());
keyValue.put(Config.CONN_DB_USERNAME, tdc.getDbUserName());
keyValue.put(Config.CONN_DB_PASSWORD, tdc.getDbPassword());
- keyValue.put(Config.CONN_WHITE_LIST, JSONObject.toJSONString(taskTopicList.get(i)));
+ keyValue.put(Config.CONN_WHITE_LIST, JSONObject.toJSONString(taskWhiteList.get(i)));
+ keyValue.put(Config.CONN_TOPIC_NAMES, taskTopicList.get(i));
keyValue.put(Config.CONN_DATA_TYPE, tdc.getDataType());
keyValue.put(Config.CONN_SOURCE_RECORD_CONVERTER, tdc.getSrcRecordConverter());
keyValue.put(Config.CONN_DB_MODE, tdc.getMode());
[rocketmq-connect] 22/43: Update Schema.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 717d4a2572d50ef8ed7703e3706a2ce4995b4484
Author: Yuchen Li <yu...@126.com>
AuthorDate: Mon Sep 9 19:21:54 2019 +0800
Update Schema.java
---
src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
index 7434bbc..93b204f 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
@@ -27,19 +27,17 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource;
-//import io.openmessaging.mysql.binlog.EventProcessor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class Schema {
-// private static final Logger LOGGER = LoggerFactory.getLogger(EventProcessor.class);
private static final String SQL = "select schema_name from information_schema.schemata";
- //取得数据库
+ //acquiring databases
private static final List<String> IGNORED_DATABASES = new ArrayList<>(
Arrays.asList(new String[] {"information_schema", "mysql", "performance_schema", "sys"})
);
- //忽略的数据库
+ //ignored databases (System Databases)
private DataSource dataSource;
public Map<String, Database> dbMap;
[rocketmq-connect] 04/43: Update Config.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 5db4b4665edfc7eb212773eb07049fad8e65c73b
Author: Yuchen Li <yu...@126.com>
AuthorDate: Fri Jul 19 14:20:24 2019 +0800
Update Config.java
---
src/main/java/org/apache/rocketmq/connect/jdbc/Config.java | 11 +----------
1 file changed, 1 insertion(+), 10 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
index 3bc3032..2b485e4 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/Config.java
@@ -42,15 +42,6 @@ public class Config {
}
};
-// public String destinationType;
-//
-// public String destinationName;
-//
-// public String messageSelector;
-//
-// private Integer sessionAcknowledgeMode = Session.AUTO_ACKNOWLEDGE;
-//
-// private Boolean sessionTransacted = Boolean.FALSE;
public void load(KeyValue props) {
@@ -134,4 +125,4 @@ public class Config {
-}
\ No newline at end of file
+}
[rocketmq-connect] 01/43: Init rocketmq-connect-jdbc
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit a9fb3af90abe73c81b8b9f3f8c64744ee86b68c1
Author: duhenglucky <du...@gmail.com>
AuthorDate: Mon Jul 15 09:32:36 2019 +0800
Init rocketmq-connect-jdbc
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..a3ebb26
--- /dev/null
+++ b/README.md
@@ -0,0 +1 @@
+# RocketMQ-connect-jdbc
[rocketmq-connect] 43/43: Add 'connector/rocketmq-connect-jdbc/' from commit '6708ada617d9f6cfef5ca42a3c2f97af44603a89'
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 24c083eccf9232b7ca7f18e0d0f236af2e68ba7f
Merge: 386adcc 6708ada
Author: odbozhou <87...@qq.com>
AuthorDate: Wed Mar 2 11:19:42 2022 +0800
Add 'connector/rocketmq-connect-jdbc/' from commit '6708ada617d9f6cfef5ca42a3c2f97af44603a89'
git-subtree-dir: connector/rocketmq-connect-jdbc
git-subtree-mainline: 386adcc43bf32d201d8ea21ba9abfc653ef4ad17
git-subtree-split: 6708ada617d9f6cfef5ca42a3c2f97af44603a89
connector/rocketmq-connect-jdbc/README.md | 85 +++++
connector/rocketmq-connect-jdbc/pom.xml | 271 ++++++++++++++++
.../rocketmq/connect/jdbc/common/CloneUtils.java | 28 ++
.../rocketmq/connect/jdbc/common/ConstDefine.java | 23 ++
.../rocketmq/connect/jdbc/common/DBUtils.java | 212 ++++++++++++
.../apache/rocketmq/connect/jdbc/common/Utils.java | 74 +++++
.../rocketmq/connect/jdbc/config/Config.java | 357 +++++++++++++++++++++
.../rocketmq/connect/jdbc/config/ConfigUtil.java | 52 +++
.../rocketmq/connect/jdbc/config/DataType.java | 26 ++
.../connect/jdbc/config/DbConnectorConfig.java | 84 +++++
.../connect/jdbc/config/SinkDbConnectorConfig.java | 98 ++++++
.../jdbc/config/SourceDbConnectorConfig.java | 73 +++++
.../connect/jdbc/config/TaskDivideConfig.java | 112 +++++++
.../connect/jdbc/config/TaskTopicInfo.java | 37 +++
.../connect/jdbc/connector/JdbcSinkConnector.java | 221 +++++++++++++
.../connect/jdbc/connector/JdbcSinkTask.java | 134 ++++++++
.../jdbc/connector/JdbcSourceConnector.java | 105 ++++++
.../connect/jdbc/connector/JdbcSourceTask.java | 183 +++++++++++
.../rocketmq/connect/jdbc/schema/Database.java | 109 +++++++
.../rocketmq/connect/jdbc/schema/Schema.java | 123 +++++++
.../apache/rocketmq/connect/jdbc/schema/Table.java | 101 ++++++
.../jdbc/schema/column/BigIntColumnParser.java | 50 +++
.../connect/jdbc/schema/column/ColumnParser.java | 104 ++++++
.../jdbc/schema/column/DateTimeColumnParser.java | 53 +++
.../jdbc/schema/column/DefaultColumnParser.java | 37 +++
.../jdbc/schema/column/EnumColumnParser.java | 46 +++
.../jdbc/schema/column/IntColumnParser.java | 66 ++++
.../jdbc/schema/column/SetColumnParser.java | 54 ++++
.../jdbc/schema/column/StringColumnParser.java | 57 ++++
.../jdbc/schema/column/TimeColumnParser.java | 39 +++
.../jdbc/schema/column/YearColumnParser.java | 40 +++
.../apache/rocketmq/connect/jdbc/sink/Updater.java | 258 +++++++++++++++
.../rocketmq/connect/jdbc/source/Querier.java | 173 ++++++++++
.../jdbc/source/TimestampIncrementingQuerier.java | 311 ++++++++++++++++++
.../connect/jdbc/strategy/DivideStrategyEnum.java | 23 ++
.../connect/jdbc/strategy/DivideTaskByQueue.java | 72 +++++
.../connect/jdbc/strategy/DivideTaskByTopic.java | 113 +++++++
.../connect/jdbc/strategy/TaskDivideStrategy.java | 32 ++
.../jdbc/connector/JdbcSourceConnectorTest.java | 79 +++++
39 files changed, 4115 insertions(+)
diff --cc connector/rocketmq-connect-jdbc/README.md
index 0000000,54c04ea..54c04ea
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/README.md
+++ b/connector/rocketmq-connect-jdbc/README.md
diff --cc connector/rocketmq-connect-jdbc/pom.xml
index 0000000,2feaa22..2feaa22
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/pom.xml
+++ b/connector/rocketmq-connect-jdbc/pom.xml
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/CloneUtils.java
index 0000000,f0ff98e..f0ff98e
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/CloneUtils.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/CloneUtils.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java
index 0000000,e6d2f7a..e6d2f7a
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/ConstDefine.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
index 0000000,963fbf6..963fbf6
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/DBUtils.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/Utils.java
index 0000000,5708e34..5708e34
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/Utils.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/Utils.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
index 0000000,1a9bbc9..1a9bbc9
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/Config.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigUtil.java
index 0000000,53563f2..53563f2
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigUtil.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/ConfigUtil.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/DataType.java
index 0000000,ef7408a..ef7408a
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/DataType.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/DataType.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/DbConnectorConfig.java
index 0000000,43bd165..43bd165
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/DbConnectorConfig.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/DbConnectorConfig.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
index 0000000,26b1541..26b1541
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/SinkDbConnectorConfig.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java
index 0000000,4972739..4972739
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/SourceDbConnectorConfig.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskDivideConfig.java
index 0000000,8b15a2f..8b15a2f
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskDivideConfig.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskDivideConfig.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskTopicInfo.java
index 0000000,5c2a21e..5c2a21e
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskTopicInfo.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/config/TaskTopicInfo.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
index 0000000,53379ec..53379ec
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkConnector.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java
index 0000000,31f43e3..31f43e3
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSinkTask.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
index 0000000,ee62133..ee62133
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnector.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index 0000000,f36623f..f36623f
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
index 0000000,33a9a22..33a9a22
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Database.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
index 0000000,1cfaf2c..1cfaf2c
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Schema.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
index 0000000,891fb9a..891fb9a
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/Table.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/BigIntColumnParser.java
index 0000000,610f07d..610f07d
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/BigIntColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/BigIntColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnParser.java
index 0000000,341064e..341064e
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/ColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java
index 0000000,c9b39e3..c9b39e3
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DateTimeColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DefaultColumnParser.java
index 0000000,ee3075a..ee3075a
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DefaultColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/DefaultColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/EnumColumnParser.java
index 0000000,0fd14ba..0fd14ba
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/EnumColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/EnumColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/IntColumnParser.java
index 0000000,36c6078..36c6078
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/IntColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/IntColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/SetColumnParser.java
index 0000000,d1e6bbc..d1e6bbc
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/SetColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/SetColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/StringColumnParser.java
index 0000000,cd4f04f..cd4f04f
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/StringColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/StringColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/TimeColumnParser.java
index 0000000,9926d81..9926d81
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/TimeColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/TimeColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/YearColumnParser.java
index 0000000,14cc798..14cc798
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/YearColumnParser.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/schema/column/YearColumnParser.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
index 0000000,9feffe6..9feffe6
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
index 0000000,03447a8..03447a8
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/Querier.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
index 0000000,0ab72df..0ab72df
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideStrategyEnum.java
index 0000000,0afa470..0afa470
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideStrategyEnum.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideStrategyEnum.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
index 0000000,9d23fd2..9d23fd2
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByQueue.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
index 0000000,5762795..5762795
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/DivideTaskByTopic.java
diff --cc connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/TaskDivideStrategy.java
index 0000000,736fcac..736fcac
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/TaskDivideStrategy.java
+++ b/connector/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/strategy/TaskDivideStrategy.java
diff --cc connector/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
index 0000000,5d25f98..5d25f98
mode 000000,100644..100644
--- a/connector/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
+++ b/connector/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceConnectorTest.java
[rocketmq-connect] 20/43: Delete JdbcSourceTaskTest.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 0521831e184355a2ae85ae9049a8572000b0ba36
Author: Yuchen Li <yu...@126.com>
AuthorDate: Fri Aug 16 14:21:26 2019 +0800
Delete JdbcSourceTaskTest.java
---
.../connect/jdbc/connector/JdbcSourceTaskTest.java | 103 ---------------------
1 file changed, 103 deletions(-)
diff --git a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java b/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java
deleted file mode 100644
index 429494b..0000000
--- a/src/test/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTaskTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.rocketmq.connect.jdbc.connector;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.sql.DataSource;
-
-import org.junit.Test;
-import java.sql.*;
-import com.alibaba.druid.pool.DruidDataSourceFactory;
-
-import io.openmessaging.KeyValue;
-import io.openmessaging.connector.api.Task;
-import io.openmessaging.connector.api.data.SourceDataEntry;
-import io.openmessaging.internal.DefaultKeyValue;
-
-public class JdbcSourceTaskTest {
- KeyValue kv;
- DataSource dataSource;
-
- @Test
- public void testBulk() throws InterruptedException {
- KeyValue kv = new DefaultKeyValue();
- kv.put("jdbcUrl", "localhost:3306");
- kv.put("jdbcUsername", "root");
- kv.put("jdbcPassword", "199812160");
- kv.put("mode", "bulk");
- kv.put("rocketmqTopic", "JdbcTopic");
- JdbcSourceTask task = new JdbcSourceTask();
- task.start(kv);
- Collection<SourceDataEntry> sourceDataEntry = task.poll();
- System.out.println(sourceDataEntry);
- }
-
- @Test
- public void testTimestampIncrementing() throws InterruptedException, SQLException {
- kv = new DefaultKeyValue();
- kv.put("jdbcUrl", "localhost:3306");
- kv.put("jdbcUsername", "root");
- kv.put("jdbcPassword", "199812160");
- kv.put("incrementingColumnName", "id");
- kv.put("timestampColmnName", "timestamp");
- kv.put("mode", "incrementing+timestamp");
- kv.put("rocketmqTopic", "JdbcTopic");
- JdbcSourceTask task = new JdbcSourceTask();
- task.start(kv);
- Collection<SourceDataEntry> sourceDataEntry = task.poll();
- System.out.println(sourceDataEntry);
- Map<String, String> map = new HashMap<>();
- map.put("driverClassName", "com.mysql.cj.jdbc.Driver");
- map.put("url", "jdbc:mysql://" + kv.getString("jdbcUrl")
- + "?useSSL=true&verifyServerCertificate=false&serverTimezone=GMT%2B8");
- map.put("username", kv.getString("jdbcUsername"));
- map.put("password", kv.getString("jdbcPassword"));
- map.put("initialSize", "2");
- map.put("maxActive", "2");
- map.put("maxWait", "60000");
- map.put("timeBetweenEvictionRunsMillis", "60000");
- map.put("minEvictableIdleTimeMillis", "300000");
- map.put("validationQuery", "SELECT 1 FROM DUAL");
- map.put("testWhileIdle", "true");
- try {
- dataSource = DruidDataSourceFactory.createDataSource(map);
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- Connection connection= dataSource.getConnection();
- PreparedStatement statement;
- String s="insert into time_db.timestamp_tb (name) values(\"test\")";
- statement=connection.prepareStatement(s);
- statement.executeUpdate();
-
- sourceDataEntry = task.poll();
- System.out.println(sourceDataEntry);
- s="update time_db.timestamp_tb set name=\"liu\" where id < 2";
- statement=connection.prepareStatement(s);
- statement.executeUpdate();
- sourceDataEntry = task.poll();
- System.out.println(sourceDataEntry);
- task.stop();
-
- connection.close();
- }
-}
[rocketmq-connect] 26/43: [ISSUE #485] Support repeat consumption (#486)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 42cbfb306735e8246f16b5e9f4be06b65cfad366
Author: chenyi19851209 <40...@qq.com>
AuthorDate: Thu Dec 19 09:22:16 2019 +0800
[ISSUE #485] Support repeat consumption (#486)
---
.../apache/rocketmq/connect/jdbc/sink/Updater.java | 74 ++++++++++++++++++++--
1 file changed, 67 insertions(+), 7 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java b/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
index 3571852..e30c65f 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/sink/Updater.java
@@ -32,18 +32,35 @@ public class Updater {
public boolean push(String dbName, String tableName, Map<Field, Object[]> fieldMap, EntryType entryType) {
Boolean isSuccess = false;
- int id = 0;
+ int beforeUpdateId = 0;
+ int afterUpdateId = 0;
switch (entryType) {
case CREATE:
- isSuccess = updateRow(dbName, tableName, fieldMap, id);
+ afterUpdateId = queryAfterUpdateRowId(dbName, tableName, fieldMap);
+ if (afterUpdateId != 0){
+ isSuccess = true;
+ break;
+ }
+ isSuccess = updateRow(dbName, tableName, fieldMap, beforeUpdateId);
break;
case UPDATE:
- id = queryRowId(dbName, tableName, fieldMap);
- isSuccess = updateRow(dbName, tableName, fieldMap, id);
+ afterUpdateId = queryAfterUpdateRowId(dbName, tableName, fieldMap);
+ if (afterUpdateId != 0){
+ isSuccess = true;
+ // 再查原有数据是否存在,存在则删除
+ beforeUpdateId = queryBeforeUpdateRowId(dbName, tableName, fieldMap);
+ if (beforeUpdateId != 0){
+ isSuccess = deleteRow(dbName, tableName, beforeUpdateId);
+ }
+ break;
+ }
+
+ beforeUpdateId = queryBeforeUpdateRowId(dbName, tableName, fieldMap);
+ isSuccess = updateRow(dbName, tableName, fieldMap, beforeUpdateId);
break;
case DELETE:
- id = queryRowId(dbName, tableName, fieldMap);
- isSuccess = deleteRow(dbName, tableName, id);
+ beforeUpdateId = queryBeforeUpdateRowId(dbName, tableName, fieldMap);
+ isSuccess = deleteRow(dbName, tableName, beforeUpdateId);
break;
default:
log.error("entryType {} is illegal.", entryType.toString());
@@ -85,7 +102,7 @@ public class Updater {
return sql;
}
- private Integer queryRowId(String dbName, String tableName, Map<Field, Object[]> fieldMap) {
+ private Integer queryBeforeUpdateRowId(String dbName, String tableName, Map<Field, Object[]> fieldMap) {
int count = 0, id = 0;
ResultSet rs;
PreparedStatement stmt;
@@ -128,6 +145,49 @@ public class Updater {
return id;
}
+ private Integer queryAfterUpdateRowId(String dbName, String tableName, Map<Field, Object[]> fieldMap) {
+ int count = 0, id = 0;
+ ResultSet rs;
+ PreparedStatement stmt;
+ Boolean finishQuery = false;
+ String query = "select id from " + dbName + "." + tableName + " where ";
+
+ for (Map.Entry<Field, Object[]> entry : fieldMap.entrySet()) {
+ count ++;
+ String fieldName = entry.getKey().getName();
+ FieldType fieldType = entry.getKey().getType();
+ Object fieldValue = entry.getValue()[1];
+ if ("id".equals(fieldName))
+ continue;
+ if (count != 1) {
+ query += " and ";
+ }
+ if (fieldValue == null)
+ {
+ query += fieldName + " is NULL";
+ } else {
+ query = typeParser(fieldType, fieldName, fieldValue, query);
+ }
+ }
+
+ try {
+ while (!connection.isClosed() && !finishQuery){
+ stmt = connection.prepareStatement(query);
+ rs = stmt.executeQuery();
+ if (rs != null) {
+ while (rs.next()) {
+ id = rs.getInt("id");
+ }
+ finishQuery = true;
+ rs.close();
+ }
+ }
+ } catch (SQLException e) {
+ log.error("query table error,{}", e);
+ }
+ return id;
+ }
+
private Boolean updateRow(String dbName, String tableName, Map<Field, Object[]> fieldMap, Integer id) {
int count = 0;
PreparedStatement stmt;
[rocketmq-connect] 18/43: Update TimestampIncrementingQuerier.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 4849ff5b56352b0782c24d314362e6e78a0cc252
Author: Yuchen Li <yu...@126.com>
AuthorDate: Fri Aug 16 14:19:49 2019 +0800
Update TimestampIncrementingQuerier.java
---
.../rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java b/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
index 1dadc4f..3201456 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/source/TimestampIncrementingQuerier.java
@@ -221,8 +221,6 @@ public class TimestampIncrementingQuerier extends Querier {
Connection conn = dataSource.getConnection();
for (Map.Entry<String, Database> entry : schema.dbMap.entrySet()) {
String db = entry.getKey();
- if (!db.contains("time_db"))
- continue;
log.info("{} database is loading", db);
Iterator<Map.Entry<String, Table>> iterator = entry.getValue().tableMap.entrySet().iterator();
while (iterator.hasNext()) {
[rocketmq-connect] 23/43: Update JdbcSourceTask.java
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 5457c06de05510322c0d8e6f25b8216b600513e1
Author: Yuchen Li <yu...@126.com>
AuthorDate: Mon Sep 9 19:27:45 2019 +0800
Update JdbcSourceTask.java
---
.../org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
index 45252bb..e01c226 100644
--- a/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
+++ b/src/main/java/org/apache/rocketmq/connect/jdbc/connector/JdbcSourceTask.java
@@ -134,7 +134,7 @@ public class JdbcSourceTask extends SourceTask {
tableQueue.add(querier);
} catch (Exception e) {
// TODO Auto-generated catch block
- e.printStackTrace();
+ log.error("Start unsuccessfully Because of {}",e);
}
} else {
TimestampIncrementingQuerier querier = new TimestampIncrementingQuerier();
@@ -144,7 +144,7 @@ public class JdbcSourceTask extends SourceTask {
tableQueue.add(querier);
} catch (Exception e) {
// TODO Auto-generated catch block
- e.printStackTrace();
+ log.error("Start unsuccessfully Because of {}",e);
}
}
[rocketmq-connect] 31/43: Update connector dependency to the latest version
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhoubo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/rocketmq-connect.git
commit 7c81b417bf150e65cd26ddafb1226710baa92ccf
Author: Han.G <gr...@gmail.com>
AuthorDate: Mon Mar 16 04:13:47 2020 +0100
Update connector dependency to the latest version
* Update pom.xml
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 59442c5..6beb19c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -189,7 +189,7 @@
<dependency>
<groupId>io.openmessaging</groupId>
<artifactId>openmessaging-connector</artifactId>
- <version>0.1.1-beta-SNAPSHOT</version>
+ <version>0.1.1</version>
</dependency>
<dependency>
<groupId>io.openmessaging</groupId>