You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by we...@apache.org on 2024/08/08 02:43:56 UTC
(inlong) branch master updated: [INLONG-10761][Agent] Delete reader related code (#10762)
This is an automated email from the ASF dual-hosted git repository.
wenweihuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new d8c2bf4c4b [INLONG-10761][Agent] Delete reader related code (#10762)
d8c2bf4c4b is described below
commit d8c2bf4c4bf9788f0bb5c6a97a8a46014cda68fe
Author: justinwwhuang <hw...@163.com>
AuthorDate: Thu Aug 8 10:43:50 2024 +0800
[INLONG-10761][Agent] Delete reader related code (#10762)
* [INLONG-10761][Agent] Delete reader related code
* [INLONG-10761][Agent] Delete reader related code
* [INLONG-10761][Agent] Delete reader related code
* [INLONG-10761][Agent] Delete reader related code
* [INLONG-10761][Agent] Delete reader related code
---
.../apache/inlong/agent/plugin/file/Reader.java | 83 ----
.../apache/inlong/agent/plugin/file/Source.java | 13 +-
.../agent/plugin/sources/DatabaseSqlSource.java | 50 ---
.../inlong/agent/plugin/sources/KafkaSource.java | 7 -
.../inlong/agent/plugin/sources/LogFileSource.java | 7 -
.../inlong/agent/plugin/sources/MongoDBSource.java | 7 -
.../inlong/agent/plugin/sources/OracleSource.java | 21 +-
.../agent/plugin/sources/PostgreSQLSource.java | 7 -
.../inlong/agent/plugin/sources/PulsarSource.java | 7 -
.../inlong/agent/plugin/sources/RedisSource.java | 13 -
.../agent/plugin/sources/SQLServerSource.java | 7 -
.../agent/plugin/sources/file/AbstractSource.java | 7 -
.../plugin/sources/reader/AbstractReader.java | 2 +-
.../agent/plugin/sources/reader/BinlogReader.java | 2 +-
.../agent/plugin/sources/reader/MongoDBReader.java | 365 ----------------
.../agent/plugin/sources/reader/MqttReader.java | 236 -----------
.../agent/plugin/sources/reader/OracleReader.java | 310 --------------
.../plugin/sources/reader/PostgreSQLReader.java | 286 -------------
.../agent/plugin/sources/reader/RedisReader.java | 458 ---------------------
.../plugin/sources/reader/SQLServerReader.java | 312 --------------
.../agent/plugin/sources/reader/SqlReader.java | 241 -----------
.../plugin/sources/reader/file/AbstractReader.java | 102 -----
.../agent/plugin/sources/PostgreSQLReaderTest.java | 102 -----
.../agent/plugin/sources/RedisReaderTest.java | 58 ---
.../agent/plugin/sources/TestBinlogReader.java | 90 ----
.../agent/plugin/sources/TestKafkaReader.java | 53 ---
.../agent/plugin/sources/TestMongoDBReader.java | 119 ------
.../agent/plugin/sources/TestMqttConnect.java | 78 ----
.../agent/plugin/sources/TestMqttReader.java | 190 ---------
.../agent/plugin/sources/TestOracleConnect.java | 64 ---
.../agent/plugin/sources/TestOracleReader.java | 235 -----------
.../agent/plugin/sources/TestSQLServerConnect.java | 66 ---
.../agent/plugin/sources/TestSQLServerReader.java | 236 -----------
33 files changed, 11 insertions(+), 3823 deletions(-)
diff --git a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Reader.java b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Reader.java
deleted file mode 100755
index 6fff49c2d3..0000000000
--- a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Reader.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.file;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.plugin.Message;
-
-/**
- * Reader reads data and provides condition whether the reading action is finished. It's called at
- * Task level.
- */
-public interface Reader {
-
- /**
- * Read message
- *
- * @return message
- */
- Message read();
-
- /**
- * Whether finish reading
- */
- boolean isFinished();
-
- /**
- * Return the reader's reading source name
- */
- String getReadSource();
-
- /**
- * set readTimeout
- */
- void setReadTimeout(long mill);
-
- /**
- * set wait milliseconds when tailing a file
- * to solve while loop cause too much cpu usage
- */
- void setWaitMillisecond(long millis);
-
- /**
- * get snapshot of the reader
- */
- String getSnapshot();
-
- /**
- * finish read
- */
- void finishRead();
-
- /**
- * source is exist
- */
- boolean isSourceExist();
-
- /**
- * init
- *
- * @param profile
- */
- void init(InstanceProfile profile);
-
- /**
- * destroy
- */
- void destroy();
-}
diff --git a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Source.java b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Source.java
index 26a5ec733d..97cb3bc181 100755
--- a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Source.java
+++ b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Source.java
@@ -18,24 +18,13 @@
package org.apache.inlong.agent.plugin.file;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.plugin.Message;
-import java.util.List;
-
/**
- * Source can be split into multiple reader.
+ * The source class is used to collect data from a data source
*/
public interface Source {
- /**
- * Split source into a list of readers.
- *
- * @param conf job conf
- * @return list of reader
- */
- List<Reader> split(TaskProfile conf);
-
/**
* Read message
*
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/DatabaseSqlSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/DatabaseSqlSource.java
index 9be71ca32f..fc86b5c8cf 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/DatabaseSqlSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/DatabaseSqlSource.java
@@ -18,17 +18,12 @@
package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
-import org.apache.inlong.agent.plugin.sources.reader.SqlReader;
-import org.apache.inlong.agent.utils.AgentDbUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
@@ -39,56 +34,11 @@ public class DatabaseSqlSource extends AbstractSource {
private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseSqlSource.class);
- private static final String JOB_DATABASE_SQL = "job.sql.command";
-
- private static final String DATABASE_SOURCE_TAG_NAME = "AgentDatabaseSourceMetric";
-
private static AtomicLong metricsIndex = new AtomicLong(0);
public DatabaseSqlSource() {
}
- /**
- * Use SQL to read data.
- *
- * @param sqlPattern sql pattern
- * @return list of readers or null if sql is not correct.
- */
- private List<Reader> splitSqlJob(String sqlPattern) {
- String[] sqlList = AgentDbUtils.replaceDynamicSeq(sqlPattern);
- if (sqlList != null) {
- List<Reader> result = new ArrayList<>();
- for (String sql : sqlList) {
- result.add(new SqlReader(sql));
- }
- return result;
- }
- return null;
- }
-
- /**
- * Use SQL or binlog to read data.
- *
- * @return reader list or null if database type is not correct.
- */
- @Override
- public List<Reader> split(TaskProfile conf) {
- String sqlPattern = conf.get(JOB_DATABASE_SQL, "").toLowerCase();
- List<Reader> readerList = null;
- if (!sqlPattern.isEmpty()) {
- readerList = splitSqlJob(sqlPattern);
- }
- if (readerList != null) {
- // increment the count of successful sources
- sourceMetric.sourceSuccessCount.incrementAndGet();
- } else {
- // database type or sql is incorrect
- // increment the count of failed sources
- sourceMetric.sourceFailCount.incrementAndGet();
- }
- return readerList;
- }
-
@Override
protected String getThreadName() {
return null;
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/KafkaSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/KafkaSource.java
index a12233bc4b..10772e6fbf 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/KafkaSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/KafkaSource.java
@@ -18,9 +18,7 @@
package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.except.FileException;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
import org.apache.commons.lang3.ObjectUtils;
@@ -168,11 +166,6 @@ public class KafkaSource extends AbstractSource {
return runnable;
}
- @Override
- public List<Reader> split(TaskProfile conf) {
- return null;
- }
-
@Override
public boolean sourceExist() {
return true;
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
index 6624ee819a..929a331777 100755
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
@@ -19,7 +19,6 @@ package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.conf.InstanceProfile;
import org.apache.inlong.agent.conf.OffsetProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.constant.DataCollectType;
import org.apache.inlong.agent.constant.TaskConstants;
import org.apache.inlong.agent.core.FileStaticManager;
@@ -27,7 +26,6 @@ import org.apache.inlong.agent.core.FileStaticManager.FileStatic;
import org.apache.inlong.agent.core.task.OffsetManager;
import org.apache.inlong.agent.except.FileException;
import org.apache.inlong.agent.metrics.audit.AuditUtils;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
import org.apache.inlong.agent.plugin.utils.file.FileDataUtils;
import org.apache.inlong.agent.utils.AgentUtils;
@@ -315,11 +313,6 @@ public class LogFileSource extends AbstractSource {
return fileExist;
}
- @Override
- public List<Reader> split(TaskProfile jobConf) {
- return null;
- }
-
@Override
protected void releaseSource() {
if (randomAccessFile != null) {
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/MongoDBSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/MongoDBSource.java
index f3f8fb7f15..6f833d7dd5 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/MongoDBSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/MongoDBSource.java
@@ -20,11 +20,9 @@ package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.common.AgentThreadFactory;
import org.apache.inlong.agent.conf.AgentConfiguration;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.constant.AgentConstants;
import org.apache.inlong.agent.constant.TaskConstants;
import org.apache.inlong.agent.except.FileException;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
import io.debezium.connector.mongodb.MongoDbConnector;
@@ -130,11 +128,6 @@ public class MongoDBSource extends AbstractSource {
committer.markBatchFinished();
}
- @Override
- public List<Reader> split(TaskProfile conf) {
- return null;
- }
-
@Override
protected String getThreadName() {
return "mongo-source-" + taskId + "-" + instanceId;
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/OracleSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/OracleSource.java
index fe7fe796ce..54faddb661 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/OracleSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/OracleSource.java
@@ -20,13 +20,10 @@ package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.common.AgentThreadFactory;
import org.apache.inlong.agent.conf.AgentConfiguration;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.constant.AgentConstants;
import org.apache.inlong.agent.except.FileException;
import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
-import org.apache.inlong.agent.plugin.sources.reader.OracleReader;
import io.debezium.connector.oracle.OracleConnector;
import io.debezium.connector.oracle.OracleConnectorConfig;
@@ -49,7 +46,14 @@ import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
-import static org.apache.inlong.agent.constant.TaskConstants.*;
+import static org.apache.inlong.agent.constant.TaskConstants.TASK_ORACLE_DBNAME;
+import static org.apache.inlong.agent.constant.TaskConstants.TASK_ORACLE_HOSTNAME;
+import static org.apache.inlong.agent.constant.TaskConstants.TASK_ORACLE_PASSWORD;
+import static org.apache.inlong.agent.constant.TaskConstants.TASK_ORACLE_PORT;
+import static org.apache.inlong.agent.constant.TaskConstants.TASK_ORACLE_SCHEMA_INCLUDE_LIST;
+import static org.apache.inlong.agent.constant.TaskConstants.TASK_ORACLE_SNAPSHOT_MODE;
+import static org.apache.inlong.agent.constant.TaskConstants.TASK_ORACLE_TABLE_INCLUDE_LIST;
+import static org.apache.inlong.agent.constant.TaskConstants.TASK_ORACLE_USER;
/**
* Oracle SQL source
@@ -71,15 +75,6 @@ public class OracleSource extends AbstractSource {
public OracleSource() {
}
- @Override
- public List<Reader> split(TaskProfile conf) {
- Reader oracleReader = new OracleReader();
- List<Reader> readerList = new ArrayList<>();
- readerList.add(oracleReader);
- sourceMetric.sourceSuccessCount.incrementAndGet();
- return readerList;
- }
-
@Override
protected String getThreadName() {
return "oracle-source-" + taskId + "-" + instanceId;
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PostgreSQLSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PostgreSQLSource.java
index 706721ce43..96f49a7f8a 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PostgreSQLSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PostgreSQLSource.java
@@ -20,11 +20,9 @@ package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.common.AgentThreadFactory;
import org.apache.inlong.agent.conf.AgentConfiguration;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.constant.AgentConstants;
import org.apache.inlong.agent.constant.TaskConstants;
import org.apache.inlong.agent.except.FileException;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
import io.debezium.connector.postgresql.PostgresConnector;
@@ -145,11 +143,6 @@ public class PostgreSQLSource extends AbstractSource {
committer.markBatchFinished();
}
- @Override
- public List<Reader> split(TaskProfile conf) {
- return null;
- }
-
@Override
protected String getThreadName() {
return "postgres-source-" + taskId + "-" + instanceId;
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PulsarSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PulsarSource.java
index 9ca7a5aaf4..56dec44d55 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PulsarSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PulsarSource.java
@@ -18,9 +18,7 @@
package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.except.FileException;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
import org.apache.commons.lang3.ObjectUtils;
@@ -165,11 +163,6 @@ public class PulsarSource extends AbstractSource {
}
}
- @Override
- public List<Reader> split(TaskProfile conf) {
- return null;
- }
-
@Override
public boolean sourceExist() {
return true;
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/RedisSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/RedisSource.java
index 25cd7a3e01..b7fea66e4c 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/RedisSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/RedisSource.java
@@ -18,16 +18,12 @@
package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
-import org.apache.inlong.agent.plugin.sources.reader.RedisReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
import java.util.List;
/**
@@ -41,15 +37,6 @@ public class RedisSource extends AbstractSource {
}
- @Override
- public List<Reader> split(TaskProfile conf) {
- RedisReader redisReader = new RedisReader();
- List<Reader> readerList = new ArrayList<>();
- readerList.add(redisReader);
- sourceMetric.sourceSuccessCount.incrementAndGet();
- return readerList;
- }
-
@Override
protected String getThreadName() {
return null;
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/SQLServerSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/SQLServerSource.java
index b6a4d6bdc0..b6ebcdd901 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/SQLServerSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/SQLServerSource.java
@@ -20,12 +20,10 @@ package org.apache.inlong.agent.plugin.sources;
import org.apache.inlong.agent.common.AgentThreadFactory;
import org.apache.inlong.agent.conf.AgentConfiguration;
import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.constant.AgentConstants;
import org.apache.inlong.agent.constant.SqlServerConstants;
import org.apache.inlong.agent.constant.TaskConstants;
import org.apache.inlong.agent.except.FileException;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
import io.debezium.connector.sqlserver.SqlServerConnector;
@@ -157,11 +155,6 @@ public class SQLServerSource extends AbstractSource {
committer.markBatchFinished();
}
- @Override
- public List<Reader> split(TaskProfile conf) {
- return null;
- }
-
@Override
protected String getThreadName() {
return "SQLServer-source-" + taskId + "-" + instanceId;
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/file/AbstractSource.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/file/AbstractSource.java
index 1fb96f7ff7..fde4cb4ff4 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/file/AbstractSource.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/file/AbstractSource.java
@@ -20,7 +20,6 @@ package org.apache.inlong.agent.plugin.sources.file;
import org.apache.inlong.agent.common.AgentThreadFactory;
import org.apache.inlong.agent.conf.InstanceProfile;
import org.apache.inlong.agent.conf.OffsetProfile;
-import org.apache.inlong.agent.conf.TaskProfile;
import org.apache.inlong.agent.constant.CycleUnitType;
import org.apache.inlong.agent.constant.TaskConstants;
import org.apache.inlong.agent.core.task.MemoryManager;
@@ -30,7 +29,6 @@ import org.apache.inlong.agent.metrics.AgentMetricItem;
import org.apache.inlong.agent.metrics.AgentMetricItemSet;
import org.apache.inlong.agent.metrics.audit.AuditUtils;
import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.file.Reader;
import org.apache.inlong.agent.plugin.file.Source;
import org.apache.inlong.agent.plugin.sources.file.extend.ExtendedHandler;
import org.apache.inlong.agent.utils.AgentUtils;
@@ -405,9 +403,4 @@ public abstract class AbstractSource implements Source {
public boolean sourceFinish() {
return emptyCount > EMPTY_CHECK_COUNT_AT_LEAST;
}
-
- @Override
- public List<Reader> split(TaskProfile conf) {
- return null;
- }
}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/AbstractReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/AbstractReader.java
index 83800619b1..829d3197bd 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/AbstractReader.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/AbstractReader.java
@@ -103,4 +103,4 @@ public abstract class AbstractReader implements Reader {
}
return new String(serializedOffset, StandardCharsets.UTF_8);
}
-}
+}
\ No newline at end of file
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/BinlogReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/BinlogReader.java
index 4153bfd121..33488366ba 100644
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/BinlogReader.java
+++ b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/BinlogReader.java
@@ -298,4 +298,4 @@ public class BinlogReader extends AbstractReader {
return AgentUtils.makeDirsIfNotExist(historyPath, parentPath).getAbsolutePath();
}
-}
+}
\ No newline at end of file
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/MongoDBReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/MongoDBReader.java
deleted file mode 100644
index 0c412cd4fe..0000000000
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/MongoDBReader.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources.reader;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.SnapshotModeConstants;
-import org.apache.inlong.agent.message.DefaultMessage;
-import org.apache.inlong.agent.metrics.audit.AuditUtils;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.file.AbstractReader;
-import org.apache.inlong.agent.plugin.sources.snapshot.MongoDBSnapshotBase;
-import org.apache.inlong.agent.plugin.utils.InLongFileOffsetBackingStore;
-import org.apache.inlong.agent.pojo.DebeziumFormat;
-import org.apache.inlong.agent.utils.GsonUtil;
-
-import com.alibaba.fastjson.JSONPath;
-import com.google.common.base.Preconditions;
-import io.debezium.config.Configuration;
-import io.debezium.config.Field;
-import io.debezium.connector.mongodb.MongoDbConnector;
-import io.debezium.engine.ChangeEvent;
-import io.debezium.engine.DebeziumEngine;
-import io.debezium.engine.format.Json;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.kafka.connect.storage.FileOffsetBackingStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.AUTO_DISCOVER_MEMBERS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.CAPTURE_MODE;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.COLLECTION_EXCLUDE_LIST;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.COLLECTION_INCLUDE_LIST;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.CONNECT_BACKOFF_INITIAL_DELAY_MS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.CONNECT_BACKOFF_MAX_DELAY_MS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.CONNECT_TIMEOUT_MS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.CURSOR_MAX_AWAIT_TIME_MS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.DATABASE_EXCLUDE_LIST;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.DATABASE_INCLUDE_LIST;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.FIELD_EXCLUDE_LIST;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.FIELD_RENAMES;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.HOSTS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.MAX_COPY_THREADS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.MAX_FAILED_CONNECTIONS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.MONGODB_POLL_INTERVAL_MS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.PASSWORD;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.SERVER_SELECTION_TIMEOUT_MS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.SNAPSHOT_MODE;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.SOCKET_TIMEOUT_MS;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.SSL_ALLOW_INVALID_HOSTNAMES;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.SSL_ENABLED;
-import static io.debezium.connector.mongodb.MongoDbConnectorConfig.USER;
-import static org.apache.inlong.agent.constant.CommonConstants.DEFAULT_MAP_CAPACITY;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_KEY_DATA;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_BACKOFF_INITIAL_DELAY;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_BACKOFF_MAX_DELAY;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_CAPTURE_MODE;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_COLLECTION_EXCLUDE_LIST;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_COLLECTION_INCLUDE_LIST;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_CONNECT_MAX_ATTEMPTS;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_CONNECT_TIMEOUT_MS;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_CURSOR_MAX_AWAIT;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_DATABASE_EXCLUDE_LIST;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_DATABASE_INCLUDE_LIST;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_FIELD_EXCLUDE_LIST;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_FIELD_RENAMES;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_HOSTS;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_INITIAL_SYNC_MAX_THREADS;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_MEMBERS_DISCOVER;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_OFFSETS;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_OFFSET_SPECIFIC_OFFSET_FILE;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_OFFSET_SPECIFIC_OFFSET_POS;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_PASSWORD;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_POLL_INTERVAL;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_QUEUE_SIZE;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_SELECTION_TIMEOUT;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_SNAPSHOT_MODE;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_SOCKET_TIMEOUT;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_SSL_ENABLE;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_SSL_INVALID_HOSTNAME_ALLOWED;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_STORE_HISTORY_FILENAME;
-import static org.apache.inlong.agent.constant.TaskConstants.TASK_MONGO_USER;
-
-/**
- * MongoDBReader : mongo source, split mongo source job into multi readers
- */
-public class MongoDBReader extends AbstractReader {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MongoDBReader.class);
-
- private String instanceId;
- private String offsetStoreFileName;
- private String specificOffsetFile;
- private String specificOffsetPos;
- private boolean finished = false;
- private boolean destroyed = false;
-
- private ExecutorService executor;
- /**
- * mongo snapshot info <br/>
- * Currently, there is no usage scenario
- */
- private MongoDBSnapshotBase snapshot;
- /**
- * message buffer queue
- */
- private LinkedBlockingQueue<Pair<String, DebeziumFormat>> bufferPool;
-
- @Override
- public Message read() {
- if (!bufferPool.isEmpty()) {
- return this.pollMessage();
- } else {
- return null;
- }
- }
-
- @Override
- public boolean isFinished() {
- return finished;
- }
-
- @Override
- public String getReadSource() {
- return instanceId;
- }
-
- public void setReadSource(String instanceId) {
- this.instanceId = instanceId;
- }
-
- @Override
- public void setReadTimeout(long mills) {
- }
-
- @Override
- public void setWaitMillisecond(long millis) {
- }
-
- @Override
- public String getSnapshot() {
- if (snapshot != null) {
- return snapshot.getSnapshot();
- } else {
- return "";
- }
- }
-
- @Override
- public void finishRead() {
- this.finished = true;
- }
-
- @Override
- public boolean isSourceExist() {
- return true;
- }
-
- @Override
- public void destroy() {
- synchronized (this) {
- if (!destroyed) {
- this.executor.shutdownNow();
- this.snapshot.close();
- this.destroyed = true;
- }
- }
- }
-
- @Override
- public void init(InstanceProfile jobConf) {
- super.init(jobConf);
- this.setGlobalParamsValue(jobConf);
- this.startEmbeddedDebeziumEngine(jobConf);
- }
-
- /**
- * poll message from buffer pool
- *
- * @return org.apache.inlong.agent.plugin.Message
- */
- private Message pollMessage() {
- // Retrieves and removes the head of this queue,
- // or returns null if this queue is empty.
- Pair<String, DebeziumFormat> message = bufferPool.poll();
- if (message == null) {
- return null;
- }
- Map<String, String> header = new HashMap<>(DEFAULT_MAP_CAPACITY);
- header.put(PROXY_KEY_DATA, message.getKey());
- return new DefaultMessage(GsonUtil.toJson(message.getValue()).getBytes(StandardCharsets.UTF_8), header);
- }
-
- /**
- * set global parameters value
- *
- * @param jobConf job conf
- */
- private void setGlobalParamsValue(InstanceProfile jobConf) {
- bufferPool = new LinkedBlockingQueue<>(jobConf.getInt(TASK_MONGO_QUEUE_SIZE, 1000));
- instanceId = jobConf.getInstanceId();
- // offset file absolute path
- offsetStoreFileName = jobConf.get(TASK_MONGO_STORE_HISTORY_FILENAME,
- MongoDBSnapshotBase.getSnapshotFilePath()) + "/mongo-" + instanceId + "-offset.dat";
- // snapshot info
- snapshot = new MongoDBSnapshotBase(offsetStoreFileName);
- String offset = jobConf.get(TASK_MONGO_OFFSETS, "");
- snapshot.save(offset, new File(offsetStoreFileName));
- // offset info
- specificOffsetFile = jobConf.get(TASK_MONGO_OFFSET_SPECIFIC_OFFSET_FILE, "");
- specificOffsetPos = jobConf.get(TASK_MONGO_OFFSET_SPECIFIC_OFFSET_POS, "-1");
- }
-
- /**
- * start the embedded debezium engine
- *
- * @param jobConf job conf
- */
- private void startEmbeddedDebeziumEngine(InstanceProfile jobConf) {
- DebeziumEngine<ChangeEvent<String, String>> debeziumEngine = DebeziumEngine.create(Json.class)
- .using(this.buildMongoConnectorConfig(jobConf))
- .notifying(this::handleChangeEvent)
- .using(this::handle)
- .build();
- this.executor = Executors.newSingleThreadExecutor();
- this.executor.execute(debeziumEngine);
- }
-
- /**
- * Handle the completion of the embedded connector engine.
- *
- * @param success {@code true} if the connector completed normally,
- * or {@code false} if the connector produced an error
- * that prevented startup or premature termination.
- * @param message the completion message; never null
- * @param error the error, or null if there was no exception
- */
- private void handle(boolean success, String message, Throwable error) {
- // jobConf.getInstanceId()
- if (!success) {
- LOGGER.error("MongoDB job with jobConf {} has error {}", message, error);
- }
- }
-
- /**
- * A Configuration object is basically a decorator around a {@link Properties} object.
- *
- * @return Configuration
- */
- private Properties buildMongoConnectorConfig(InstanceProfile jobConf) {
- Configuration.Builder builder = Configuration.create();
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_HOSTS, HOSTS);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_USER, USER);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_PASSWORD, PASSWORD);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_DATABASE_INCLUDE_LIST, DATABASE_INCLUDE_LIST);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_DATABASE_EXCLUDE_LIST, DATABASE_EXCLUDE_LIST);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_COLLECTION_INCLUDE_LIST, COLLECTION_INCLUDE_LIST);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_COLLECTION_EXCLUDE_LIST, COLLECTION_EXCLUDE_LIST);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_FIELD_EXCLUDE_LIST, FIELD_EXCLUDE_LIST);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_SNAPSHOT_MODE, SNAPSHOT_MODE);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_CAPTURE_MODE, CAPTURE_MODE);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_CONNECT_TIMEOUT_MS, CONNECT_TIMEOUT_MS);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_CURSOR_MAX_AWAIT, CURSOR_MAX_AWAIT_TIME_MS);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_SOCKET_TIMEOUT, SOCKET_TIMEOUT_MS);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_SELECTION_TIMEOUT, SERVER_SELECTION_TIMEOUT_MS);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_FIELD_RENAMES, FIELD_RENAMES);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_MEMBERS_DISCOVER, AUTO_DISCOVER_MEMBERS);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_CONNECT_MAX_ATTEMPTS, MAX_FAILED_CONNECTIONS);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_BACKOFF_MAX_DELAY, CONNECT_BACKOFF_MAX_DELAY_MS);
- setEngineConfigIfNecessary(jobConf, builder,
- TASK_MONGO_BACKOFF_INITIAL_DELAY, CONNECT_BACKOFF_INITIAL_DELAY_MS);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_INITIAL_SYNC_MAX_THREADS, MAX_COPY_THREADS);
- setEngineConfigIfNecessary(jobConf, builder,
- TASK_MONGO_SSL_INVALID_HOSTNAME_ALLOWED, SSL_ALLOW_INVALID_HOSTNAMES);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_SSL_ENABLE, SSL_ENABLED);
- setEngineConfigIfNecessary(jobConf, builder, TASK_MONGO_POLL_INTERVAL, MONGODB_POLL_INTERVAL_MS);
-
- Properties props = builder.build().asProperties();
- props.setProperty("offset.storage.file.filename", offsetStoreFileName);
- props.setProperty("connector.class", MongoDbConnector.class.getCanonicalName());
- props.setProperty("name", "engine-" + instanceId);
- props.setProperty("mongodb.name", "inlong-mongodb-" + instanceId);
-
- String snapshotMode = props.getOrDefault(TASK_MONGO_SNAPSHOT_MODE, "").toString();
- if (Objects.equals(SnapshotModeConstants.INITIAL, snapshotMode)) {
- Preconditions.checkNotNull(TASK_MONGO_OFFSET_SPECIFIC_OFFSET_FILE,
- TASK_MONGO_OFFSET_SPECIFIC_OFFSET_FILE + " cannot be null");
- Preconditions.checkNotNull(TASK_MONGO_OFFSET_SPECIFIC_OFFSET_POS,
- TASK_MONGO_OFFSET_SPECIFIC_OFFSET_POS + " cannot be null");
- props.setProperty("offset.storage", InLongFileOffsetBackingStore.class.getCanonicalName());
- props.setProperty(InLongFileOffsetBackingStore.OFFSET_STATE_VALUE,
- serializeOffset(instanceId, specificOffsetFile, specificOffsetPos));
- } else {
- props.setProperty("offset.storage", FileOffsetBackingStore.class.getCanonicalName());
- }
- LOGGER.info("mongo job {} start with props {}",
- jobConf.getInstanceId(),
- GsonUtil.toJson(props));
- return props;
- }
-
- private void setEngineConfigIfNecessary(InstanceProfile jobConf,
- Configuration.Builder builder, String key, Field field) {
- String value = jobConf.get(key, field.defaultValueAsString());
- if (StringUtils.isBlank(value)) {
- return;
- }
- builder.with(field, value);
- }
-
- /**
- * Handles a batch of records, calling the {@link DebeziumEngine.RecordCommitter#markProcessed(Object)}
- * for each record and {@link DebeziumEngine.RecordCommitter#markBatchFinished()} when this batch is finished.
- *
- * @param records the records to be processed
- * @param committer the committer that indicates to the system that we are finished
- */
- private void handleChangeEvent(List<ChangeEvent<String, String>> records,
- DebeziumEngine.RecordCommitter<ChangeEvent<String, String>> committer) {
- try {
- for (ChangeEvent<String, String> record : records) {
- DebeziumFormat debeziumFormat = JSONPath.read(record.value(), "$.payload", DebeziumFormat.class);
- bufferPool.put(Pair.of(debeziumFormat.getSource().getCollection(), debeziumFormat));
- committer.markProcessed(record);
- }
- committer.markBatchFinished();
- long dataSize = records.stream().mapToLong(c -> c.value().length()).sum();
- AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS, super.inlongGroupId, super.inlongStreamId,
- System.currentTimeMillis(), records.size(), dataSize);
- readerMetric.pluginReadSuccessCount.addAndGet(records.size());
- readerMetric.pluginReadCount.addAndGet(records.size());
- } catch (InterruptedException e) {
- e.printStackTrace();
- LOGGER.error("parse mongo message error", e);
- readerMetric.pluginReadFailCount.addAndGet(records.size());
- readerMetric.pluginReadCount.addAndGet(records.size());
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/MqttReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/MqttReader.java
deleted file mode 100644
index 45660748c2..0000000000
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/MqttReader.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources.reader;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.TaskConstants;
-import org.apache.inlong.agent.message.DefaultMessage;
-import org.apache.inlong.agent.metrics.audit.AuditUtils;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.file.AbstractReader;
-
-import org.apache.commons.lang3.StringUtils;
-import org.eclipse.paho.client.mqttv3.IMqttDeliveryToken;
-import org.eclipse.paho.client.mqttv3.MqttCallback;
-import org.eclipse.paho.client.mqttv3.MqttClient;
-import org.eclipse.paho.client.mqttv3.MqttConnectOptions;
-import org.eclipse.paho.client.mqttv3.MqttException;
-import org.eclipse.paho.client.mqttv3.MqttMessage;
-import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.LinkedBlockingQueue;
-
-public class MqttReader extends AbstractReader {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(MqttReader.class);
-
- private boolean finished = false;
-
- private boolean destroyed = false;
-
- private MqttClient client;
-
- private MqttConnectOptions options;
-
- private String serverURI;
- private String userName;
- private String password;
- private String topic;
- private int qos;
- private boolean cleanSession = false;
- private boolean automaticReconnect = true;
- private InstanceProfile jobProfile;
- private String instanceId;
- private String clientId;
- private int mqttVersion = MqttConnectOptions.MQTT_VERSION_DEFAULT;
-
- private LinkedBlockingQueue<DefaultMessage> mqttMessagesQueue;
-
- public MqttReader(String topic) {
- this.topic = topic;
- }
-
- /**
- * set global params value
- *
- * @param jobConf
- */
- private void setGlobalParamsValue(InstanceProfile jobConf) {
- mqttMessagesQueue = new LinkedBlockingQueue<>(jobConf.getInt(TaskConstants.TASK_MQTT_QUEUE_SIZE, 1000));
- instanceId = jobConf.getInstanceId();
- userName = jobConf.get(TaskConstants.TASK_MQTT_USERNAME);
- password = jobConf.get(TaskConstants.TASK_MQTT_PASSWORD);
- serverURI = jobConf.get(TaskConstants.TASK_MQTT_SERVER_URI);
- clientId = jobConf.get(TaskConstants.TASK_MQTT_CLIENT_ID_PREFIX, "mqtt_client") + "_" + UUID.randomUUID();
- cleanSession = jobConf.getBoolean(TaskConstants.TASK_MQTT_CLEAN_SESSION, false);
- automaticReconnect = jobConf.getBoolean(TaskConstants.TASK_MQTT_AUTOMATIC_RECONNECT, true);
- qos = jobConf.getInt(TaskConstants.TASK_MQTT_QOS, 1);
- mqttVersion = jobConf.getInt(TaskConstants.TASK_MQTT_VERSION, MqttConnectOptions.MQTT_VERSION_DEFAULT);
- options = new MqttConnectOptions();
- options.setCleanSession(cleanSession);
- options.setConnectionTimeout(jobConf.getInt(TaskConstants.TASK_MQTT_CONNECTION_TIMEOUT, 10));
- options.setKeepAliveInterval(jobConf.getInt(TaskConstants.TASK_MQTT_KEEPALIVE_INTERVAL, 20));
- options.setUserName(userName);
- options.setPassword(password.toCharArray());
- options.setAutomaticReconnect(automaticReconnect);
- options.setMqttVersion(mqttVersion);
- }
-
- /**
- * connect to MQTT Broker
- */
- private void connect() {
-
- try {
- synchronized (MqttReader.class) {
- client = new MqttClient(serverURI, clientId, new MemoryPersistence());
- client.setCallback(new MqttCallback() {
-
- @Override
- public void connectionLost(Throwable cause) {
- LOGGER.error("the mqtt jobId:{}, serverURI:{}, connection lost, {} ", instanceId,
- serverURI, cause);
- reconnect();
- }
-
- @Override
- public void messageArrived(String topic, MqttMessage message) throws Exception {
- Map<String, String> headerMap = new HashMap<>();
- headerMap.put("record.topic", topic);
- headerMap.put("record.messageId", String.valueOf(message.getId()));
- headerMap.put("record.qos", String.valueOf(message.getQos()));
- byte[] recordValue = message.getPayload();
- mqttMessagesQueue.put(new DefaultMessage(recordValue, headerMap));
-
- AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS, inlongGroupId, inlongStreamId,
- System.currentTimeMillis(), 1, recordValue.length);
-
- readerMetric.pluginReadSuccessCount.incrementAndGet();
- readerMetric.pluginReadCount.incrementAndGet();
- }
-
- @Override
- public void deliveryComplete(IMqttDeliveryToken token) {
- }
- });
- client.connect(options);
- client.subscribe(topic, qos);
- }
- LOGGER.info("the mqtt subscribe topic is [{}], qos is [{}]", topic, qos);
- } catch (Exception e) {
- LOGGER.error("init mqtt client error {}. jobId:{},serverURI:{},clientId:{}", e, instanceId, serverURI,
- clientId);
- }
- }
-
- @Override
- public void init(InstanceProfile jobConf) {
- super.init(jobConf);
- jobProfile = jobConf;
- LOGGER.info("init mqtt reader with jobConf {}", jobConf.toJsonStr());
- setGlobalParamsValue(jobConf);
- connect();
- }
-
- private void reconnect() {
- if (!client.isConnected()) {
- try {
- client.connect(options);
- LOGGER.info("the mqtt client reconnect success. jobId:{}, serverURI:{}, clientId:{}", instanceId,
- serverURI, clientId);
- } catch (Exception e) {
- LOGGER.error("reconnect mqtt client error {}. jobId:{}, serverURI:{}, clientId:{}", e, instanceId,
- serverURI, clientId);
- }
- }
- }
-
- @Override
- public Message read() {
- if (!mqttMessagesQueue.isEmpty()) {
- return getMqttMessage();
- } else {
- return null;
- }
- }
-
- private DefaultMessage getMqttMessage() {
- return mqttMessagesQueue.poll();
- }
-
- @Override
- public boolean isFinished() {
- return finished;
- }
-
- @Override
- public String getReadSource() {
- return instanceId;
- }
-
- @Override
- public void setReadTimeout(long mill) {
- }
-
- @Override
- public void setWaitMillisecond(long millis) {
- }
-
- @Override
- public String getSnapshot() {
- return StringUtils.EMPTY;
- }
-
- @Override
- public void finishRead() {
- finished = true;
- }
-
- @Override
- public boolean isSourceExist() {
- return true;
- }
-
- private void disconnect() {
- try {
- client.disconnect();
- } catch (MqttException e) {
- LOGGER.error("disconnect mqtt client error {}. jobId:{},serverURI:{},clientId:{}", e, instanceId, serverURI,
- clientId);
- }
- }
-
- public void setReadSource(String instanceId) {
- this.instanceId = instanceId;
- }
-
- @Override
- public void destroy() {
- synchronized (this) {
- if (!destroyed) {
- disconnect();
- destroyed = true;
- }
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/OracleReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/OracleReader.java
deleted file mode 100644
index 0bb92f2951..0000000000
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/OracleReader.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources.reader;
-
-import org.apache.inlong.agent.conf.AgentConfiguration;
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.AgentConstants;
-import org.apache.inlong.agent.constant.OracleConstants;
-import org.apache.inlong.agent.constant.SnapshotModeConstants;
-import org.apache.inlong.agent.message.DefaultMessage;
-import org.apache.inlong.agent.metrics.audit.AuditUtils;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.file.AbstractReader;
-import org.apache.inlong.agent.plugin.sources.snapshot.OracleSnapshotBase;
-import org.apache.inlong.agent.plugin.utils.InLongDatabaseHistory;
-import org.apache.inlong.agent.plugin.utils.InLongFileOffsetBackingStore;
-import org.apache.inlong.agent.pojo.DebeziumFormat;
-import org.apache.inlong.agent.utils.AgentUtils;
-import org.apache.inlong.agent.utils.GsonUtil;
-
-import com.google.common.base.Preconditions;
-import com.google.gson.Gson;
-import io.debezium.connector.oracle.OracleConnector;
-import io.debezium.engine.ChangeEvent;
-import io.debezium.engine.DebeziumEngine;
-import io.debezium.relational.history.FileDatabaseHistory;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.kafka.connect.storage.FileOffsetBackingStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.apache.inlong.agent.constant.CommonConstants.DEFAULT_MAP_CAPACITY;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_KEY_DATA;
-
-/**
- * Read data from Oracle database by Debezium
- */
-public class OracleReader extends AbstractReader {
-
- public static final String ORACLE_READER_TAG_NAME = "AgentOracleMetric";
- public static final String JOB_DATABASE_USER = "job.oracleJob.user";
- public static final String JOB_DATABASE_PASSWORD = "job.oracleJob.password";
- public static final String JOB_DATABASE_HOSTNAME = "job.oracleJob.hostname";
- public static final String JOB_DATABASE_PORT = "job.oracleJob.port";
- public static final String JOB_DATABASE_SNAPSHOT_MODE = "job.oracleJob.snapshot.mode";
- public static final String JOB_DATABASE_SERVER_NAME = "job.oracleJob.serverName";
- public static final String JOB_DATABASE_QUEUE_SIZE = "job.oracleJob.queueSize";
- public static final String JOB_DATABASE_OFFSETS = "job.oracleJob.offsets";
- public static final String JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE = "job.oracleJob.offset.specificOffsetFile";
- public static final String JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS = "job.oracleJob.offset.specificOffsetPos";
- public static final String JOB_DATABASE_DBNAME = "job.oracleJob.dbname";
- public static final String JOB_DATABASE_STORE_OFFSET_INTERVAL_MS = "job.oracleJob.offset.intervalMs";
- public static final String JOB_DATABASE_STORE_HISTORY_FILENAME = "job.oracleJob.history.filename";
-
- private static final Gson GSON = new Gson();
- public static final String ORACLE = "oracle";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SqlReader.class);
-
- private final AgentConfiguration agentConf = AgentConfiguration.getAgentConf();
-
- private String databaseStoreHistoryName;
- private String instanceId;
- private String dbName;
- private String serverName;
- private String userName;
- private String password;
- private String hostName;
- private String port;
- private String offsetFlushIntervalMs;
- private String offsetStoreFileName;
- private String snapshotMode;
- private String offset;
- private String specificOffsetFile;
- private String specificOffsetPos;
- private OracleSnapshotBase oracleSnapshot;
- private boolean finished = false;
- private ExecutorService executor;
-
- /**
- * pair.left : table name
- * pair.right : actual data
- */
- private LinkedBlockingQueue<Pair<String, String>> oracleMessageQueue;
- private InstanceProfile jobProfile;
- private boolean destroyed = false;
-
- public OracleReader() {
- }
-
- @Override
- public Message read() {
- if (!oracleMessageQueue.isEmpty()) {
- return getOracleMessage();
- } else {
- return null;
- }
- }
-
- /**
- * poll message from buffer pool
- *
- * @return org.apache.inlong.agent.plugin.Message
- */
- private DefaultMessage getOracleMessage() {
- // Retrieves and removes the head of this queue,
- // or returns null if this queue is empty.
- Pair<String, String> message = oracleMessageQueue.poll();
- if (Objects.isNull(message)) {
- return null;
- }
- Map<String, String> header = new HashMap<>(DEFAULT_MAP_CAPACITY);
- header.put(PROXY_KEY_DATA, message.getKey());
- return new DefaultMessage(GsonUtil.toJson(message.getValue()).getBytes(StandardCharsets.UTF_8), header);
- }
-
- public boolean isDestroyed() {
- return destroyed;
- }
-
- @Override
- public boolean isFinished() {
- return finished;
- }
-
- @Override
- public String getReadSource() {
- return instanceId;
- }
-
- @Override
- public void setReadTimeout(long mill) {
-
- }
-
- @Override
- public void setWaitMillisecond(long millis) {
-
- }
-
- @Override
- public String getSnapshot() {
- if (oracleSnapshot != null) {
- return oracleSnapshot.getSnapshot();
- } else {
- return StringUtils.EMPTY;
- }
- }
-
- @Override
- public void finishRead() {
- this.finished = true;
- }
-
- @Override
- public boolean isSourceExist() {
- return true;
- }
-
- private String tryToInitAndGetHistoryPath() {
- String historyPath = agentConf.get(
- AgentConstants.AGENT_HISTORY_PATH, AgentConstants.DEFAULT_AGENT_HISTORY_PATH);
- String parentPath = agentConf.get(
- AgentConstants.AGENT_HOME, AgentConstants.DEFAULT_AGENT_HOME);
- return AgentUtils.makeDirsIfNotExist(historyPath, parentPath).getAbsolutePath();
- }
-
- @Override
- public void init(InstanceProfile jobConf) {
- super.init(jobConf);
- jobProfile = jobConf;
- LOGGER.info("init oracle reader with jobConf {}", jobConf.toJsonStr());
- userName = jobConf.get(JOB_DATABASE_USER);
- password = jobConf.get(JOB_DATABASE_PASSWORD);
- hostName = jobConf.get(JOB_DATABASE_HOSTNAME);
- port = jobConf.get(JOB_DATABASE_PORT);
- dbName = jobConf.get(JOB_DATABASE_DBNAME);
- serverName = jobConf.get(JOB_DATABASE_SERVER_NAME);
- instanceId = jobConf.getInstanceId();
- offsetFlushIntervalMs = jobConf.get(JOB_DATABASE_STORE_OFFSET_INTERVAL_MS, "100000");
- offsetStoreFileName = jobConf.get(JOB_DATABASE_STORE_HISTORY_FILENAME,
- tryToInitAndGetHistoryPath()) + "/offset.dat" + instanceId;
- snapshotMode = jobConf.get(JOB_DATABASE_SNAPSHOT_MODE, OracleConstants.INITIAL);
- oracleMessageQueue = new LinkedBlockingQueue<>(jobConf.getInt(JOB_DATABASE_QUEUE_SIZE, 1000));
- finished = false;
-
- databaseStoreHistoryName = jobConf.get(JOB_DATABASE_STORE_HISTORY_FILENAME,
- tryToInitAndGetHistoryPath()) + "/history.dat" + jobConf.getInstanceId();
- offset = jobConf.get(JOB_DATABASE_OFFSETS, "");
- specificOffsetFile = jobConf.get(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE, "");
- specificOffsetPos = jobConf.get(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS, "-1");
-
- oracleSnapshot = new OracleSnapshotBase(offsetStoreFileName);
- oracleSnapshot.save(offset, oracleSnapshot.getFile());
-
- Properties props = getEngineProps();
-
- DebeziumEngine<ChangeEvent<String, String>> engine = DebeziumEngine.create(
- io.debezium.engine.format.Json.class)
- .using(props)
- .notifying((records, committer) -> {
- try {
- for (ChangeEvent<String, String> record : records) {
- DebeziumFormat debeziumFormat = GSON
- .fromJson(record.value(), DebeziumFormat.class);
- oracleMessageQueue.put(Pair.of(debeziumFormat.getSource().getTable(), record.value()));
- committer.markProcessed(record);
- }
- committer.markBatchFinished();
- long dataSize = records.stream().mapToLong(c -> c.value().length()).sum();
- AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS, inlongGroupId, inlongStreamId,
- System.currentTimeMillis(), records.size(), dataSize);
- readerMetric.pluginReadSuccessCount.addAndGet(records.size());
- readerMetric.pluginReadCount.addAndGet(records.size());
- } catch (Exception e) {
- readerMetric.pluginReadFailCount.addAndGet(records.size());
- readerMetric.pluginReadCount.addAndGet(records.size());
- LOGGER.error("parse binlog message error", e);
- }
- })
- .using((success, message, error) -> {
- if (!success) {
- LOGGER.error("oracle job with jobConf {} has error {}", instanceId, message, error);
- }
- }).build();
-
- executor = Executors.newSingleThreadExecutor();
- executor.execute(engine);
-
- LOGGER.info("get initial snapshot of job {}, snapshot {}", instanceId, getSnapshot());
- }
-
- private Properties getEngineProps() {
- Properties props = new Properties();
- props.setProperty("name", "engine" + instanceId);
- props.setProperty("connector.class", OracleConnector.class.getCanonicalName());
- props.setProperty("database.hostname", hostName);
- props.setProperty("database.port", port);
- props.setProperty("database.user", userName);
- props.setProperty("database.password", password);
- props.setProperty("database.dbname", dbName);
- props.setProperty("database.server.name", serverName);
- props.setProperty("offset.flush.interval.ms", offsetFlushIntervalMs);
- props.setProperty("database.snapshot.mode", snapshotMode);
- props.setProperty("key.converter.schemas.enable", "false");
- props.setProperty("value.converter.schemas.enable", "false");
- props.setProperty("snapshot.mode", snapshotMode);
- props.setProperty("offset.storage.file.filename", offsetStoreFileName);
- props.setProperty("database.history.file.filename", databaseStoreHistoryName);
- if (SnapshotModeConstants.SPECIFIC_OFFSETS.equals(snapshotMode)) {
- Preconditions.checkNotNull(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE,
- JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE + " cannot be null");
- Preconditions.checkNotNull(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS,
- JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS + " cannot be null");
- props.setProperty("offset.storage", InLongFileOffsetBackingStore.class.getCanonicalName());
- props.setProperty(InLongFileOffsetBackingStore.OFFSET_STATE_VALUE,
- serializeOffset(instanceId, specificOffsetFile, specificOffsetPos));
- props.setProperty("database.history", InLongDatabaseHistory.class.getCanonicalName());
- } else {
- props.setProperty("offset.storage", FileOffsetBackingStore.class.getCanonicalName());
- props.setProperty("database.history", FileDatabaseHistory.class.getCanonicalName());
- }
- props.setProperty("tombstones.on.delete", "false");
- props.setProperty("converters", "datetime");
- props.setProperty("datetime.type", "org.apache.inlong.agent.plugin.utils.BinlogTimeConverter");
- props.setProperty("datetime.format.date", "yyyy-MM-dd");
- props.setProperty("datetime.format.time", "HH:mm:ss");
- props.setProperty("datetime.format.datetime", "yyyy-MM-dd HH:mm:ss");
- props.setProperty("datetime.format.timestamp", "yyyy-MM-dd HH:mm:ss");
-
- LOGGER.info("oracle job {} start with props {}", jobProfile.getInstanceId(), props);
- return props;
- }
-
- @Override
- public void destroy() {
- synchronized (this) {
- if (!destroyed) {
- this.executor.shutdownNow();
- this.oracleSnapshot.close();
- this.destroyed = true;
- }
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/PostgreSQLReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/PostgreSQLReader.java
deleted file mode 100644
index aa54265f1f..0000000000
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/PostgreSQLReader.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources.reader;
-
-import org.apache.inlong.agent.conf.AgentConfiguration;
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.AgentConstants;
-import org.apache.inlong.agent.constant.PostgreSQLConstants;
-import org.apache.inlong.agent.message.DefaultMessage;
-import org.apache.inlong.agent.metrics.audit.AuditUtils;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.file.AbstractReader;
-import org.apache.inlong.agent.plugin.sources.snapshot.PostgreSQLSnapshotBase;
-import org.apache.inlong.agent.plugin.utils.InLongFileOffsetBackingStore;
-import org.apache.inlong.agent.pojo.DebeziumFormat;
-import org.apache.inlong.agent.utils.AgentUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.gson.Gson;
-import io.debezium.connector.postgresql.PostgresConnector;
-import io.debezium.engine.ChangeEvent;
-import io.debezium.engine.DebeziumEngine;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.kafka.connect.storage.FileOffsetBackingStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.apache.inlong.agent.constant.CommonConstants.DEFAULT_MAP_CAPACITY;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_KEY_DATA;
-
-/**
- * Read postgreSQL data
- */
-public class PostgreSQLReader extends AbstractReader {
-
- public static final String COMPONENT_NAME = "PostgreSQLReader";
- public static final String JOB_POSTGRESQL_USER = "job.postgreSQLJob.user";
- public static final String JOB_DATABASE_PASSWORD = "job.postgreSQLJob.password";
- public static final String JOB_DATABASE_HOSTNAME = "job.postgreSQLJob.hostname";
- public static final String JOB_DATABASE_PORT = "job.postgreSQLJob.port";
- public static final String JOB_DATABASE_STORE_OFFSET_INTERVAL_MS = "job.postgreSQLJob.offset.intervalMs";
- public static final String JOB_DATABASE_STORE_HISTORY_FILENAME = "job.postgreSQLJob.history.filename";
- public static final String JOB_DATABASE_SNAPSHOT_MODE = "job.postgreSQLJob.snapshot.mode";
- public static final String JOB_DATABASE_QUEUE_SIZE = "job.postgreSQLJob.queueSize";
- public static final String JOB_DATABASE_OFFSETS = "job.postgreSQLJob.offsets";
- public static final String JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE = "job.postgreSQLJob.offset.specificOffsetFile";
- public static final String JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS = "job.postgreSQLJob.offset.specificOffsetPos";
- public static final String JOB_DATABASE_DBNAME = "job.postgreSQLJob.dbname";
- public static final String JOB_DATABASE_SERVER_NAME = "job.postgreSQLJob.servername";
- public static final String JOB_DATABASE_PLUGIN_NAME = "job.postgreSQLJob.pluginname";
- private static final Gson GSON = new Gson();
- private static final Logger LOGGER = LoggerFactory.getLogger(PostgreSQLReader.class);
- private final AgentConfiguration agentConf = AgentConfiguration.getAgentConf();
- private String userName;
- private String password;
- private String hostName;
- private String port;
- private String offsetFlushIntervalMs;
- private String offsetStoreFileName;
- private String snapshotMode;
- private String instanceId;
- private String offset;
- private String specificOffsetFile;
- private String specificOffsetPos;
- private String dbName;
- private String pluginName;
- private String serverName;
- private PostgreSQLSnapshotBase postgreSQLSnapshot;
- private boolean finished = false;
- private ExecutorService executor;
- /**
- * pair.left : table name
- * pair.right : actual data
- */
- private LinkedBlockingQueue<Pair<String, String>> postgreSQLMessageQueue;
- private InstanceProfile jobProfile;
- private boolean destroyed = false;
-
- public PostgreSQLReader() {
- }
-
- @Override
- public Message read() {
- if (!postgreSQLMessageQueue.isEmpty()) {
- return getPostgreSQLMessage();
- } else {
- return null;
- }
- }
-
- private DefaultMessage getPostgreSQLMessage() {
- Pair<String, String> message = postgreSQLMessageQueue.poll();
- Map<String, String> header = new HashMap<>(DEFAULT_MAP_CAPACITY);
- header.put(PROXY_KEY_DATA, message.getKey());
- return new DefaultMessage(message.getValue().getBytes(StandardCharsets.UTF_8), header);
- }
-
- public void init(InstanceProfile jobConf) {
- super.init(jobConf);
- jobProfile = jobConf;
- LOGGER.info("init PostgreSQL reader with jobConf {}", jobConf.toJsonStr());
- userName = jobConf.get(JOB_POSTGRESQL_USER);
- password = jobConf.get(JOB_DATABASE_PASSWORD);
- hostName = jobConf.get(JOB_DATABASE_HOSTNAME);
- port = jobConf.get(JOB_DATABASE_PORT);
- dbName = jobConf.get(JOB_DATABASE_DBNAME);
- serverName = jobConf.get(JOB_DATABASE_SERVER_NAME);
- pluginName = jobConf.get(JOB_DATABASE_PLUGIN_NAME, "pgoutput");
- instanceId = jobConf.getInstanceId();
- offsetFlushIntervalMs = jobConf.get(JOB_DATABASE_STORE_OFFSET_INTERVAL_MS, "100000");
- offsetStoreFileName = jobConf.get(JOB_DATABASE_STORE_HISTORY_FILENAME,
- tryToInitAndGetHistoryPath()) + "/offset.dat" + instanceId;
- snapshotMode = jobConf.get(JOB_DATABASE_SNAPSHOT_MODE, PostgreSQLConstants.INITIAL);
- postgreSQLMessageQueue = new LinkedBlockingQueue<>(jobConf.getInt(JOB_DATABASE_QUEUE_SIZE, 1000));
- finished = false;
-
- offset = jobConf.get(JOB_DATABASE_OFFSETS, "");
- specificOffsetFile = jobConf.get(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE, "");
- specificOffsetPos = jobConf.get(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS, "-1");
- postgreSQLSnapshot = new PostgreSQLSnapshotBase(offsetStoreFileName);
- postgreSQLSnapshot.save(offset, postgreSQLSnapshot.getFile());
-
- Properties props = getEngineProps();
-
- DebeziumEngine<ChangeEvent<String, String>> engine = DebeziumEngine.create(
- io.debezium.engine.format.Json.class)
- .using(props)
- .notifying((records, committer) -> {
- try {
- for (ChangeEvent<String, String> record : records) {
- DebeziumFormat debeziumFormat = GSON
- .fromJson(record.value(), DebeziumFormat.class);
- postgreSQLMessageQueue.put(Pair.of(debeziumFormat.getSource().getTable(), record.value()));
- committer.markProcessed(record);
- }
- committer.markBatchFinished();
- long dataSize = records.stream().mapToLong(c -> c.value().length()).sum();
- AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS, inlongGroupId, inlongStreamId,
- System.currentTimeMillis(), records.size(), dataSize);
- readerMetric.pluginReadSuccessCount.addAndGet(records.size());
- readerMetric.pluginReadCount.addAndGet(records.size());
- } catch (Exception e) {
- readerMetric.pluginReadFailCount.addAndGet(records.size());
- readerMetric.pluginReadCount.addAndGet(records.size());
- LOGGER.error("parse binlog message error", e);
- }
- })
- .using((success, message, error) -> {
- if (!success) {
- LOGGER.error("PostgreSQL job with jobConf {} has error {}", instanceId, message, error);
- }
- }).build();
-
- executor = Executors.newSingleThreadExecutor();
- executor.execute(engine);
-
- LOGGER.info("get initial snapshot of job {}, snapshot {}", instanceId, getSnapshot());
- }
-
- private String tryToInitAndGetHistoryPath() {
- String historyPath = agentConf.get(
- AgentConstants.AGENT_HISTORY_PATH, AgentConstants.DEFAULT_AGENT_HISTORY_PATH);
- String parentPath = agentConf.get(
- AgentConstants.AGENT_HOME, AgentConstants.DEFAULT_AGENT_HOME);
- return AgentUtils.makeDirsIfNotExist(historyPath, parentPath).getAbsolutePath();
- }
-
- private Properties getEngineProps() {
- Properties props = new Properties();
-
- props.setProperty("name", "engine" + instanceId);
- props.setProperty("connector.class", PostgresConnector.class.getCanonicalName());
- props.setProperty("database.server.name", serverName);
- props.setProperty("plugin.name", pluginName);
- props.setProperty("slot.name", "slot" + instanceId);
- props.setProperty("database.hostname", hostName);
- props.setProperty("database.port", port);
- props.setProperty("database.user", userName);
- props.setProperty("database.dbname", dbName);
- props.setProperty("database.password", password);
-
- props.setProperty("offset.flush.interval.ms", offsetFlushIntervalMs);
- props.setProperty("database.snapshot.mode", snapshotMode);
- props.setProperty("key.converter.schemas.enable", "false");
- props.setProperty("value.converter.schemas.enable", "false");
- props.setProperty("snapshot.mode", snapshotMode);
- props.setProperty("offset.storage.file.filename", offsetStoreFileName);
- if (PostgreSQLConstants.CUSTOM.equals(snapshotMode)) {
- Preconditions.checkNotNull(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE,
- JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE + " cannot be null");
- Preconditions.checkNotNull(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS,
- JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS + " cannot be null");
- props.setProperty("offset.storage", InLongFileOffsetBackingStore.class.getCanonicalName());
- props.setProperty(InLongFileOffsetBackingStore.OFFSET_STATE_VALUE,
- serializeOffset(instanceId, specificOffsetFile, specificOffsetPos));
- } else {
- props.setProperty("offset.storage", FileOffsetBackingStore.class.getCanonicalName());
- }
- props.setProperty("tombstones.on.delete", "false");
- props.setProperty("converters", "datetime");
- props.setProperty("datetime.type", "org.apache.inlong.agent.plugin.utils.BinlogTimeConverter");
- props.setProperty("datetime.format.date", "yyyy-MM-dd");
- props.setProperty("datetime.format.time", "HH:mm:ss");
- props.setProperty("datetime.format.datetime", "yyyy-MM-dd HH:mm:ss");
- props.setProperty("datetime.format.timestamp", "yyyy-MM-dd HH:mm:ss");
-
- LOGGER.info("PostgreSQL job {} start with props {}", jobProfile.getInstanceId(), props);
- return props;
- }
-
- @Override
- public void destroy() {
- synchronized (this) {
- if (!destroyed) {
- executor.shutdownNow();
- postgreSQLSnapshot.close();
- destroyed = true;
- }
- }
- }
-
- @Override
- public boolean isFinished() {
- return finished;
- }
-
- @Override
- public String getReadSource() {
- return instanceId;
- }
-
- public void setReadSource(String instanceId) {
- this.instanceId = instanceId;
- }
-
- @Override
- public void setReadTimeout(long mill) {
- return;
- }
-
- @Override
- public void setWaitMillisecond(long millis) {
- return;
- }
-
- @Override
- public String getSnapshot() {
- if (postgreSQLSnapshot != null) {
- return postgreSQLSnapshot.getSnapshot();
- }
- return "";
- }
-
- @Override
- public void finishRead() {
- finished = true;
- }
-
- @Override
- public boolean isSourceExist() {
- return true;
- }
-}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/RedisReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/RedisReader.java
deleted file mode 100644
index 9c95b0206f..0000000000
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/RedisReader.java
+++ /dev/null
@@ -1,458 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources.reader;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.message.DefaultMessage;
-import org.apache.inlong.agent.metrics.audit.AuditUtils;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.file.AbstractReader;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.TypeAdapter;
-import com.google.gson.stream.JsonReader;
-import com.google.gson.stream.JsonWriter;
-import com.moilioncircle.redis.replicator.RedisReplicator;
-import com.moilioncircle.redis.replicator.Replicator;
-import com.moilioncircle.redis.replicator.cmd.CommandName;
-import com.moilioncircle.redis.replicator.cmd.impl.DefaultCommand;
-import com.moilioncircle.redis.replicator.cmd.parser.DefaultCommandParser;
-import com.moilioncircle.redis.replicator.event.Event;
-import com.moilioncircle.redis.replicator.event.EventListener;
-import com.moilioncircle.redis.replicator.event.PostRdbSyncEvent;
-import com.moilioncircle.redis.replicator.rdb.datatype.KeyStringValueHash;
-import com.moilioncircle.redis.replicator.rdb.datatype.KeyStringValueList;
-import com.moilioncircle.redis.replicator.rdb.datatype.KeyStringValueSet;
-import com.moilioncircle.redis.replicator.rdb.datatype.KeyStringValueString;
-import com.moilioncircle.redis.replicator.rdb.datatype.KeyStringValueZSet;
-import com.moilioncircle.redis.replicator.rdb.datatype.KeyValuePair;
-import com.moilioncircle.redis.replicator.rdb.datatype.ZSetEntry;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-
-/**
- * Redis data reader
- */
-public class RedisReader extends AbstractReader {
-
- public static final String REDIS_READER_TAG_NAME = "AgentRedisMetric";
- public static final String JOB_REDIS_PORT = "job.redisJob.port";
- public static final String JOB_REDIS_HOSTNAME = "job.redisJob.hostname";
- public static final String JOB_REDIS_SSL = "job.redisJob.ssl";
- public static final String JOB_REDIS_AUTHUSER = "job.redisJob.authUser";
- public static final String JOB_REDIS_AUTHPASSWORD = "job.redisJob.authPassword";
- public static final String JOB_REDIS_READTIMEOUT = "job.redisJob.readTimeout";
- public static final String JOB_REDIS_QUEUE_SIZE = "job.redisJob.queueSize";
- public static final String JOB_REDIS_REPLID = "job.redisJob.replId";
- public static final String JOB_REDIS_OFFSET = "job.redisJob.offset";
- private static final Logger LOGGER = LoggerFactory.getLogger(RedisReader.class);
- private String port;
- private String hostName;
- private boolean ssl;
- private String authUser;
- private String authPassword;
- private String readTimeout;
- private String instanceId;
- private String replId;
- private String snapShot;
- private boolean destroyed;
- private Replicator redisReplicator;
- private LinkedBlockingQueue<String> redisMessageQueue;
- private boolean finished = false;
- private ExecutorService executor;
- private Gson gson;
-
- @Override
- public void init(InstanceProfile jobConf) {
- LOGGER.info("Init redis reader with jobConf {}", jobConf.toJsonStr());
- port = jobConf.get(JOB_REDIS_PORT);
- hostName = jobConf.get(JOB_REDIS_HOSTNAME);
- ssl = jobConf.getBoolean(JOB_REDIS_SSL, false);
- authUser = jobConf.get(JOB_REDIS_AUTHUSER, "");
- authPassword = jobConf.get(JOB_REDIS_AUTHPASSWORD, "");
- readTimeout = jobConf.get(JOB_REDIS_READTIMEOUT, "");
- replId = jobConf.get(JOB_REDIS_REPLID, "");
- snapShot = jobConf.get(JOB_REDIS_OFFSET, "-1");
- instanceId = jobConf.getInstanceId();
- finished = false;
- redisMessageQueue = new LinkedBlockingQueue<>(jobConf.getInt(JOB_REDIS_QUEUE_SIZE, 10000));
- initGson();
- String uri = getRedisUri();
- try {
- redisReplicator = new RedisReplicator(uri);
- initReplicator();
- redisReplicator.addEventListener(new EventListener() {
-
- @Override
- public void onEvent(Replicator replicator, Event event) {
- try {
- if (event instanceof DefaultCommand || event instanceof KeyValuePair<?, ?>) {
- String eventJson = gson.toJson(event);
- redisMessageQueue.put(eventJson);
- AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS, inlongGroupId, inlongStreamId,
- System.currentTimeMillis(), 1, eventJson.length());
- readerMetric.pluginReadCount.incrementAndGet();
- }
- if (event instanceof PostRdbSyncEvent) {
- snapShot = String.valueOf(replicator.getConfiguration().getReplOffset());
- LOGGER.info("after rdb snapShot is: {}", snapShot);
- }
- } catch (InterruptedException e) {
- readerMetric.pluginReadFailCount.incrementAndGet();
- LOGGER.error("Read redis data error", e);
- }
- }
- });
- executor = Executors.newSingleThreadExecutor();
- executor.execute(new Thread(() -> {
- try {
- redisReplicator.open();
- } catch (IOException e) {
- LOGGER.error("Redis source error", e);
- }
- }));
- } catch (URISyntaxException | IOException e) {
- readerMetric.pluginReadFailCount.addAndGet(1);
- LOGGER.error("Connect to redis {}:{} failed.", hostName, port);
- }
- }
-
- private String getRedisUri() {
- StringBuffer sb = new StringBuffer("redis://");
- sb.append(hostName).append(":").append(port);
- sb.append("?");
- if (!StringUtils.isEmpty(authPassword)) {
- sb.append("authPassword=").append(authPassword).append("&");
- }
- if (!StringUtils.isEmpty(authUser)) {
- sb.append("authUser=").append(authUser).append("&");
- }
- if (!StringUtils.isEmpty(readTimeout)) {
- sb.append("readTimeout=").append(readTimeout).append("&");
- }
- if (ssl) {
- sb.append("ssl=").append("yes").append("&");
- }
- if (!StringUtils.isEmpty(snapShot)) {
- sb.append("replOffset=").append(snapShot).append("&");
- }
- if (!StringUtils.isEmpty(replId)) {
- sb.append("replId=").append(replId).append("&");
- }
- if (sb.charAt(sb.length() - 1) == '?' || sb.charAt(sb.length() - 1) == '&') {
- sb.deleteCharAt(sb.length() - 1);
- }
- return sb.toString();
- }
-
- @Override
- public void destroy() {
- synchronized (this) {
- if (!destroyed) {
- try {
- executor.shutdown();
- redisReplicator.close();
- } catch (IOException e) {
- LOGGER.error("Redis reader close failed.");
- }
- destroyed = true;
- }
- }
- }
-
- @Override
- public Message read() {
- if (!redisMessageQueue.isEmpty()) {
- readerMetric.pluginReadCount.incrementAndGet();
- return new DefaultMessage(redisMessageQueue.poll().getBytes());
- } else {
- return null;
- }
- }
-
- @Override
- public boolean isFinished() {
- return finished;
- }
-
- @Override
- public String getReadSource() {
- return instanceId;
- }
-
- public void setReadSource(String instanceId) {
- this.instanceId = instanceId;
- }
-
- @Override
- public void setReadTimeout(long mill) {
-
- }
-
- @Override
- public void setWaitMillisecond(long millis) {
-
- }
-
- @Override
- public String getSnapshot() {
- return snapShot;
- }
-
- @Override
- public void finishRead() {
- finished = true;
- }
-
- @Override
- public boolean isSourceExist() {
- return true;
- }
-
- /**
- * init GSON parser
- */
- private void initGson() {
- gson = new GsonBuilder().registerTypeAdapter(KeyStringValueHash.class, new TypeAdapter<KeyStringValueHash>() {
-
- @Override
- public void write(JsonWriter out, KeyStringValueHash kv) throws IOException {
- out.beginObject();
- out.name("DB").beginObject();
- out.name("dbNumber").value(kv.getDb().getDbNumber());
- out.name("dbSize").value(kv.getDb().getDbsize());
- out.name("expires").value(kv.getDb().getExpires());
- out.endObject();
- out.name("valueRdbType").value(kv.getValueRdbType());
- out.name("key").value(new String(kv.getKey()));
- out.name("value").beginObject();
- for (byte[] b : kv.getValue().keySet()) {
- out.name(new String(b)).value(new String(kv.getValue().get(b)));
- }
- out.endObject();
- out.endObject();
- }
-
- @Override
- public KeyStringValueHash read(JsonReader in) throws IOException {
- return null;
- }
- }).registerTypeAdapter(DefaultCommand.class, new TypeAdapter<DefaultCommand>() {
-
- @Override
- public void write(JsonWriter out, DefaultCommand dc) throws IOException {
- out.beginObject();
- out.name("key").value(new String(dc.getCommand()));
- out.name("value").beginArray();
- for (byte[] bytes : dc.getArgs()) {
- out.value(new String(bytes));
- }
- out.endArray();
- out.endObject();
- }
-
- @Override
- public DefaultCommand read(JsonReader in) throws IOException {
- return null;
- }
- })
- .registerTypeAdapter(KeyStringValueList.class, new TypeAdapter<KeyStringValueList>() {
-
- @Override
- public void write(JsonWriter out, KeyStringValueList kv) throws IOException {
- out.beginObject();
- out.name("key").value(new String(kv.getKey()));
- out.name("value").beginArray();
- for (byte[] bytes : kv.getValue()) {
- out.value(new String(bytes));
- }
- out.endArray();
- out.endObject();
- }
-
- @Override
- public KeyStringValueList read(JsonReader in) throws IOException {
- return null;
- }
- })
- .registerTypeAdapter(KeyStringValueSet.class, new TypeAdapter<KeyStringValueSet>() {
-
- @Override
- public void write(JsonWriter out, KeyStringValueSet kv) throws IOException {
- out.beginObject();
- out.name("key").value(new String(kv.getKey()));
- out.name("value").beginArray();
- for (byte[] bytes : kv.getValue()) {
- out.value(new String(bytes));
- }
- out.endArray();
- out.endObject();
- }
-
- @Override
- public KeyStringValueSet read(JsonReader in) throws IOException {
- return null;
- }
- })
- .registerTypeAdapter(KeyStringValueString.class, new TypeAdapter<KeyStringValueString>() {
-
- @Override
- public void write(JsonWriter out, KeyStringValueString kv) throws IOException {
- out.beginObject();
- out.name("key").value(new String(kv.getKey()));
- out.name("value").value(new String(kv.getValue()));
- out.endObject();
- }
-
- @Override
- public KeyStringValueString read(JsonReader in) throws IOException {
- return null;
- }
- })
- .registerTypeAdapter(KeyStringValueZSet.class, new TypeAdapter<KeyStringValueZSet>() {
-
- @Override
- public void write(JsonWriter out, KeyStringValueZSet kv) throws IOException {
- out.beginObject();
- out.name("key").value(new String(kv.getKey()));
- out.name("value").beginArray();
- for (ZSetEntry entry : kv.getValue()) {
- out.beginObject();
- out.name("element").value(new String(entry.getElement()));
- out.name("score").value(entry.getScore());
- out.endObject();
- }
- out.endArray();
- out.endObject();
- }
-
- @Override
- public KeyStringValueZSet read(JsonReader in) throws IOException {
- return null;
- }
- })
- .create();
- }
-
- /**
- * init replicator's commandParser
- */
- private void initReplicator() {
- DefaultCommandParser defaultCommandParser = new DefaultCommandParser();
- redisReplicator.addCommandParser(CommandName.name("APPEND"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SET"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SETEX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("MSET"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("DEL"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SADD"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("HMSET"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("HSET"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("LSET"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("EXPIRE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("EXPIREAT"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("GETSET"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("HSETNX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("MSETNX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("PSETEX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SETNX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SETRANGE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("HDEL"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("LPOP"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("LPUSH"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("LPUSHX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("LRem"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("RPOP"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("RPUSH"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("RPUSHX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZREM"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("RENAME"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("INCR"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("DECR"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("INCRBY"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("DECRBY"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("PERSIST"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SELECT"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("FLUSHALL"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("FLUSHDB"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("HINCRBY"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZINCRBY"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("MOVE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SMOVE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("PFADD"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("PFCOUNT"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("PFMERGE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SDIFFSTORE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SINTERSTORE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SUNIONSTORE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZADD"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZINTERSTORE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZUNIONSTORE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("BRPOPLPUSH"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("LINSERT"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("RENAMENX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("RESTORE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("PEXPIRE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("PEXPIREAT"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("GEOADD"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("EVAL"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("EVALSHA"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SCRIPT"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("PUBLISH"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("BITOP"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("BITFIELD"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SETBIT"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SREM"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("UNLINK"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SWAPDB"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("MULTI"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("EXEC"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZREMRANGEBYSCORE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZREMRANGEBYRANK"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZREMRANGEBYLEX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("LTRIM"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("SORT"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("RPOPLPUSH"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZPOPMIN"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZPOPMAX"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("REPLCONF"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("XACK"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("XADD"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("XCLAIM"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("XDEL"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("XGROUP"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("XTRIM"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("XSETID"), defaultCommandParser);
- // since redis 6.2
- redisReplicator.addCommandParser(CommandName.name("COPY"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("LMOVE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("BLMOVE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("ZDIFFSTORE"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("GEOSEARCHSTORE"), defaultCommandParser);
- // since redis 7.0
- redisReplicator.addCommandParser(CommandName.name("SPUBLISH"), defaultCommandParser);
- redisReplicator.addCommandParser(CommandName.name("FUNCTION"), defaultCommandParser);
- }
-}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/SQLServerReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/SQLServerReader.java
deleted file mode 100644
index 6a211872d5..0000000000
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/SQLServerReader.java
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources.reader;
-
-import org.apache.inlong.agent.conf.AgentConfiguration;
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.AgentConstants;
-import org.apache.inlong.agent.constant.SnapshotModeConstants;
-import org.apache.inlong.agent.constant.SqlServerConstants;
-import org.apache.inlong.agent.message.DefaultMessage;
-import org.apache.inlong.agent.metrics.audit.AuditUtils;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.file.AbstractReader;
-import org.apache.inlong.agent.plugin.sources.snapshot.SqlServerSnapshotBase;
-import org.apache.inlong.agent.plugin.utils.InLongDatabaseHistory;
-import org.apache.inlong.agent.plugin.utils.InLongFileOffsetBackingStore;
-import org.apache.inlong.agent.pojo.DebeziumFormat;
-import org.apache.inlong.agent.utils.AgentUtils;
-import org.apache.inlong.agent.utils.GsonUtil;
-
-import com.google.common.base.Preconditions;
-import com.google.gson.Gson;
-import io.debezium.connector.sqlserver.SqlServerConnector;
-import io.debezium.engine.ChangeEvent;
-import io.debezium.engine.DebeziumEngine;
-import io.debezium.relational.history.FileDatabaseHistory;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.kafka.connect.storage.FileOffsetBackingStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import static org.apache.inlong.agent.constant.CommonConstants.DEFAULT_MAP_CAPACITY;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_KEY_DATA;
-
-/**
- * Read data from SQLServer database by Debezium
- */
-public class SQLServerReader extends AbstractReader {
-
- public static final String SQLSERVER_READER_TAG_NAME = "AgentSQLServerMetric";
- public static final String JOB_DATABASE_HOSTNAME = "job.sqlserverJob.hostname";
- public static final String JOB_DATABASE_PORT = "job.sqlserverJob.port";
- public static final String JOB_DATABASE_USER = "job.sqlserverJob.user";
- public static final String JOB_DATABASE_PASSWORD = "job.sqlserverJob.password";
- public static final String JOB_DATABASE_DBNAME = "job.sqlserverJob.dbname";
- public static final String JOB_DATABASE_SNAPSHOT_MODE = "job.sqlserverJob.snapshot.mode";
- public static final String JOB_DATABASE_QUEUE_SIZE = "job.sqlserverJob.queueSize";
- public static final String JOB_DATABASE_OFFSETS = "job.sqlserverJob.offsets";
- public static final String JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE = "job.sqlserverJob.offset.specificOffsetFile";
- public static final String JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS = "job.sqlserverJob.offset.specificOffsetPos";
-
- public static final String JOB_DATABASE_SERVER_NAME = "job.sqlserverJob.serverName";
-
- public static final String JOB_DATABASE_STORE_OFFSET_INTERVAL_MS = "job.sqlserverJob.offset.intervalMs";
- public static final String JOB_DATABASE_STORE_HISTORY_FILENAME = "job.sqlserverJob.history.filename";
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SQLServerReader.class);
- private final AgentConfiguration agentConf = AgentConfiguration.getAgentConf();
-
- private static final Gson GSON = new Gson();
-
- private String databaseStoreHistoryName;
- private String instanceId;
- private String dbName;
- private String serverName;
- private String userName;
- private String password;
- private String hostName;
- private String port;
- private String offsetFlushIntervalMs;
- private String offsetStoreFileName;
- private String snapshotMode;
- private String offset;
- private String specificOffsetFile;
- private String specificOffsetPos;
-
- private ExecutorService executor;
- private SqlServerSnapshotBase sqlServerSnapshot;
- private boolean finished = false;
-
- private LinkedBlockingQueue<Pair<String, String>> sqlServerMessageQueue;
- private InstanceProfile jobProfile;
- private boolean destroyed = false;
-
- public SQLServerReader() {
-
- }
-
- @Override
- public Message read() {
- if (!sqlServerMessageQueue.isEmpty()) {
- return getSqlServerMessage();
- } else {
- return null;
- }
- }
-
- /**
- * poll message from buffer pool
- *
- * @return org.apache.inlong.agent.plugin.Message
- */
- private DefaultMessage getSqlServerMessage() {
- // Retrieves and removes the head of this queue,
- // or returns null if this queue is empty.
- Pair<String, String> message = sqlServerMessageQueue.poll();
- if (Objects.isNull(message)) {
- return null;
- }
- Map<String, String> header = new HashMap<>(DEFAULT_MAP_CAPACITY);
- header.put(PROXY_KEY_DATA, message.getKey());
- return new DefaultMessage(GsonUtil.toJson(message.getValue()).getBytes(StandardCharsets.UTF_8), header);
- }
-
- public boolean isDestroyed() {
- return destroyed;
- }
-
- @Override
- public boolean isFinished() {
- return finished;
- }
-
- @Override
- public String getReadSource() {
- return instanceId;
- }
-
- public void setReadSource(String instanceId) {
- this.instanceId = instanceId;
- }
-
- @Override
- public void setReadTimeout(long mill) {
-
- }
-
- @Override
- public void setWaitMillisecond(long millis) {
-
- }
-
- @Override
- public String getSnapshot() {
- if (sqlServerSnapshot != null) {
- return sqlServerSnapshot.getSnapshot();
- } else {
- return StringUtils.EMPTY;
- }
- }
-
- @Override
- public void finishRead() {
- this.finished = true;
- }
-
- @Override
- public boolean isSourceExist() {
- return true;
- }
-
- private String tryToInitAndGetHistoryPath() {
- String historyPath = agentConf.get(
- AgentConstants.AGENT_HISTORY_PATH, AgentConstants.DEFAULT_AGENT_HISTORY_PATH);
- String parentPath = agentConf.get(
- AgentConstants.AGENT_HOME, AgentConstants.DEFAULT_AGENT_HOME);
- return AgentUtils.makeDirsIfNotExist(historyPath, parentPath).getAbsolutePath();
- }
-
- @Override
- public void init(InstanceProfile jobConf) {
- super.init(jobConf);
- jobProfile = jobConf;
- LOGGER.info("init SqlServer reader with jobConf {}", jobConf.toJsonStr());
- userName = jobConf.get(JOB_DATABASE_USER);
- password = jobConf.get(JOB_DATABASE_PASSWORD);
- hostName = jobConf.get(JOB_DATABASE_HOSTNAME);
- port = jobConf.get(JOB_DATABASE_PORT);
- dbName = jobConf.get(JOB_DATABASE_DBNAME);
- serverName = jobConf.get(JOB_DATABASE_SERVER_NAME);
- instanceId = jobConf.getInstanceId();
- offsetFlushIntervalMs = jobConf.get(JOB_DATABASE_STORE_OFFSET_INTERVAL_MS, "100000");
- offsetStoreFileName = jobConf.get(JOB_DATABASE_STORE_HISTORY_FILENAME,
- tryToInitAndGetHistoryPath()) + "/offset.dat" + instanceId;
- snapshotMode = jobConf.get(JOB_DATABASE_SNAPSHOT_MODE, SqlServerConstants.INITIAL);
- sqlServerMessageQueue = new LinkedBlockingQueue<>(jobConf.getInt(JOB_DATABASE_QUEUE_SIZE, 1000));
- finished = false;
-
- databaseStoreHistoryName = jobConf.get(JOB_DATABASE_STORE_HISTORY_FILENAME,
- tryToInitAndGetHistoryPath()) + "/history.dat" + jobConf.getInstanceId();
- offset = jobConf.get(JOB_DATABASE_OFFSETS, "");
- specificOffsetFile = jobConf.get(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE, "");
- specificOffsetPos = jobConf.get(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS, "-1");
-
- sqlServerSnapshot = new SqlServerSnapshotBase(offsetStoreFileName);
- sqlServerSnapshot.save(offset, sqlServerSnapshot.getFile());
-
- Properties props = getEngineProps();
-
- DebeziumEngine<ChangeEvent<String, String>> engine = DebeziumEngine.create(
- io.debezium.engine.format.Json.class)
- .using(props)
- .notifying((records, committer) -> {
- try {
- for (ChangeEvent<String, String> record : records) {
- DebeziumFormat debeziumFormat = GSON
- .fromJson(record.value(), DebeziumFormat.class);
- sqlServerMessageQueue.put(Pair.of(debeziumFormat.getSource().getTable(), record.value()));
- committer.markProcessed(record);
- }
- committer.markBatchFinished();
- long dataSize = records.stream().mapToLong(c -> c.value().length()).sum();
- AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS, inlongGroupId, inlongStreamId,
- System.currentTimeMillis(), records.size(), dataSize);
- readerMetric.pluginReadSuccessCount.addAndGet(records.size());
- readerMetric.pluginReadCount.addAndGet(records.size());
- } catch (Exception e) {
- readerMetric.pluginReadFailCount.addAndGet(records.size());
- readerMetric.pluginReadCount.addAndGet(records.size());
- LOGGER.error("parse SqlServer message error", e);
- }
- })
- .using((success, message, error) -> {
- if (!success) {
- LOGGER.error("SqlServer job with jobConf {} has error {}", instanceId, message, error);
- }
- }).build();
-
- executor = Executors.newSingleThreadExecutor();
- executor.execute(engine);
-
- LOGGER.info("get initial snapshot of job {}, snapshot {}", instanceId, getSnapshot());
- }
-
- private Properties getEngineProps() {
- Properties props = new Properties();
- props.setProperty("name", "engine-" + instanceId);
- props.setProperty("connector.class", SqlServerConnector.class.getCanonicalName());
- props.setProperty("database.hostname", hostName);
- props.setProperty("database.port", port);
- props.setProperty("database.user", userName);
- props.setProperty("database.password", password);
- props.setProperty("database.dbname", dbName);
- props.setProperty("database.server.name", serverName);
- props.setProperty("offset.flush.interval.ms", offsetFlushIntervalMs);
- props.setProperty("database.snapshot.mode", snapshotMode);
- props.setProperty("key.converter.schemas.enable", "false");
- props.setProperty("value.converter.schemas.enable", "false");
- props.setProperty("snapshot.mode", snapshotMode);
- props.setProperty("offset.storage.file.filename", offsetStoreFileName);
- props.setProperty("database.history.file.filename", databaseStoreHistoryName);
- if (SnapshotModeConstants.SPECIFIC_OFFSETS.equals(snapshotMode)) {
- Preconditions.checkNotNull(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE,
- JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE + " cannot be null");
- Preconditions.checkNotNull(JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS,
- JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS + " cannot be null");
- props.setProperty("offset.storage", InLongFileOffsetBackingStore.class.getCanonicalName());
- props.setProperty(InLongFileOffsetBackingStore.OFFSET_STATE_VALUE,
- serializeOffset(instanceId, specificOffsetFile, specificOffsetPos));
- props.setProperty("database.history", InLongDatabaseHistory.class.getCanonicalName());
- } else {
- props.setProperty("offset.storage", FileOffsetBackingStore.class.getCanonicalName());
- props.setProperty("database.history", FileDatabaseHistory.class.getCanonicalName());
- }
- props.setProperty("tombstones.on.delete", "false");
- props.setProperty("converters", "datetime");
- props.setProperty("datetime.type", "org.apache.inlong.agent.plugin.utils.BinlogTimeConverter");
- props.setProperty("datetime.format.date", "yyyy-MM-dd");
- props.setProperty("datetime.format.time", "HH:mm:ss");
- props.setProperty("datetime.format.datetime", "yyyy-MM-dd HH:mm:ss");
- props.setProperty("datetime.format.timestamp", "yyyy-MM-dd HH:mm:ss");
-
- LOGGER.info("SqlServer job {} start with props {}", jobProfile.getInstanceId(), props);
- return props;
- }
-
- @Override
- public void destroy() {
- synchronized (this) {
- if (!destroyed) {
- this.executor.shutdownNow();
- this.sqlServerSnapshot.close();
- this.destroyed = true;
- }
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/SqlReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/SqlReader.java
deleted file mode 100755
index e41bfbe82c..0000000000
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/SqlReader.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources.reader;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.message.DefaultMessage;
-import org.apache.inlong.agent.metrics.audit.AuditUtils;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.file.AbstractReader;
-import org.apache.inlong.agent.utils.AgentDbUtils;
-import org.apache.inlong.agent.utils.AgentUtils;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang3.CharUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.charset.StandardCharsets;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-
-import static java.sql.Types.BINARY;
-import static java.sql.Types.BLOB;
-import static java.sql.Types.LONGVARBINARY;
-import static java.sql.Types.VARBINARY;
-
-/**
- * Read data from database by SQL
- */
-public class SqlReader extends AbstractReader {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(SqlReader.class);
-
- private static final String SQL_READER_TAG_NAME = "AgentSqlMetric";
-
- private static final String JOB_DATABASE_USER = "job.sql.user";
- private static final String JOB_DATABASE_PASSWORD = "job.sql.password";
- private static final String JOB_DATABASE_HOSTNAME = "job.sql.hostname";
- private static final String JOB_DATABASE_PORT = "job.sql.port";
-
- private static final String JOB_DATABASE_BATCH_SIZE = "job.sql.batchSize";
- private static final int DEFAULT_JOB_DATABASE_BATCH_SIZE = 1000;
-
- private static final String JOB_DATABASE_DRIVER_CLASS = "job.database.driverClass";
- private static final String DEFAULT_JOB_DATABASE_DRIVER_CLASS = "com.mysql.jdbc.Driver";
-
- // pre-set sql lines, commands like "set xxx=xx;"
- private static final String JOB_DATABASE_TYPE = "job.database.type";
- private static final String MYSQL = "mysql";
-
- /* Standard short field separator */
- private static final String STD_FIELD_SEPARATOR_SHORT = "\001";
- private static final String JOB_DATABASE_SEPARATOR = "job.sql.separator";
- private static final String[] NEW_LINE_CHARS = new String[]{String.valueOf(CharUtils.CR),
- String.valueOf(CharUtils.LF)};
- private static final String[] EMPTY_CHARS = new String[]{StringUtils.EMPTY, StringUtils.EMPTY};
-
- private final String sql;
-
- // use statement for mysql due to compatibility
- private Statement statement;
- private PreparedStatement preparedStatement;
- private Connection conn;
- private ResultSet resultSet;
- private int columnCount;
-
- // column types
- private String[] columnTypeNames;
- private int[] columnTypeCodes;
- private boolean finished = false;
- private String separator;
-
- public SqlReader(String sql) {
- this.sql = sql;
- }
-
- @Override
- public Message read() {
- try {
- if (resultSet.next()) {
- List<String> lineColumns = new ArrayList<>();
- for (int i = 1; i <= columnCount; i++) {
- String dataValue = null;
- /* handle special blob value, encode with base64, BLOB=2004 */
- int typeCode = columnTypeCodes[i - 1];
- String typeName = columnTypeNames[i - 1];
-
- // binary type
- if (typeCode == BLOB || typeCode == BINARY || typeCode == VARBINARY
- || typeCode == LONGVARBINARY || typeName.contains("BLOB")) {
- byte[] data = resultSet.getBytes(i);
- dataValue = new String(Base64.encodeBase64(data, false), StandardCharsets.UTF_8);
- } else {
- // non-binary type
- dataValue = StringUtils.replaceEachRepeatedly(resultSet.getString(i),
- NEW_LINE_CHARS, EMPTY_CHARS);
- }
- lineColumns.add(dataValue);
- }
- long dataSize = lineColumns.stream().mapToLong(column -> column.length()).sum();
- AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS,
- inlongGroupId, inlongStreamId, System.currentTimeMillis(), 1, dataSize);
- readerMetric.pluginReadSuccessCount.incrementAndGet();
- readerMetric.pluginReadCount.incrementAndGet();
- return generateMessage(lineColumns);
- } else {
- finished = true;
- }
- } catch (Exception ex) {
- LOGGER.error("error while reading data", ex);
- readerMetric.pluginReadFailCount.incrementAndGet();
- readerMetric.pluginReadCount.incrementAndGet();
- throw new RuntimeException(ex);
- }
- return null;
- }
-
- private Message generateMessage(List<String> lineColumns) {
- return new DefaultMessage(StringUtils.join(lineColumns, separator).getBytes(StandardCharsets.UTF_8));
- }
-
- @Override
- public boolean isFinished() {
- return finished;
- }
-
- @Override
- public String getReadSource() {
- return sql;
- }
-
- @Override
- public void setReadTimeout(long mill) {
-
- }
-
- @Override
- public void setWaitMillisecond(long millis) {
-
- }
-
- @Override
- public String getSnapshot() {
- return StringUtils.EMPTY;
- }
-
- @Override
- public void finishRead() {
- destroy();
- }
-
- @Override
- public boolean isSourceExist() {
- return true;
- }
-
- /**
- * Init column meta data.
- *
- * @throws Exception - sql exception
- */
- private void initColumnMeta() throws Exception {
- columnCount = resultSet.getMetaData().getColumnCount();
- columnTypeNames = new String[columnCount];
- columnTypeCodes = new int[columnCount];
- for (int i = 0; i < columnCount; i++) {
- columnTypeCodes[i] = resultSet.getMetaData().getColumnType(i + 1);
- String t = resultSet.getMetaData().getColumnTypeName(i + 1);
- if (t != null) {
- columnTypeNames[i] = t.toUpperCase();
- }
- }
- }
-
- @Override
- public void init(InstanceProfile jobConf) {
- super.init(jobConf);
- int batchSize = jobConf.getInt(JOB_DATABASE_BATCH_SIZE, DEFAULT_JOB_DATABASE_BATCH_SIZE);
- String userName = jobConf.get(JOB_DATABASE_USER);
- String password = jobConf.get(JOB_DATABASE_PASSWORD);
- String hostName = jobConf.get(JOB_DATABASE_HOSTNAME);
- int port = jobConf.getInt(JOB_DATABASE_PORT);
-
- String driverClass = jobConf.get(JOB_DATABASE_DRIVER_CLASS,
- DEFAULT_JOB_DATABASE_DRIVER_CLASS);
- separator = jobConf.get(JOB_DATABASE_SEPARATOR, STD_FIELD_SEPARATOR_SHORT);
- finished = false;
- try {
- String databaseType = jobConf.get(JOB_DATABASE_TYPE, MYSQL);
- String url = String.format("jdbc:%s://%s:%d", databaseType, hostName, port);
- conn = AgentDbUtils.getConnectionFailover(
- driverClass, url, userName, password);
- if (databaseType.equals(MYSQL)) {
- statement = conn.createStatement(
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- statement.setFetchSize(Integer.MIN_VALUE);
- resultSet = statement.executeQuery(sql);
- } else {
- // better performance in non-mysql server
- preparedStatement = conn.prepareStatement(sql);
- preparedStatement.setFetchSize(batchSize);
- resultSet = preparedStatement.executeQuery();
- }
- initColumnMeta();
- } catch (Exception ex) {
- LOGGER.error("error create statement", ex);
- destroy();
- throw new RuntimeException(ex);
- }
-
- }
-
- @Override
- public void destroy() {
- finished = true;
- AgentUtils.finallyClose(resultSet);
- AgentUtils.finallyClose(statement);
- AgentUtils.finallyClose(preparedStatement);
- AgentUtils.finallyClose(conn);
- }
-}
diff --git a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/AbstractReader.java b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/AbstractReader.java
deleted file mode 100644
index 611c29945b..0000000000
--- a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/reader/file/AbstractReader.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources.reader.file;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.metrics.AgentMetricItem;
-import org.apache.inlong.agent.metrics.AgentMetricItemSet;
-import org.apache.inlong.agent.plugin.file.Reader;
-import org.apache.inlong.agent.pojo.DebeziumOffset;
-import org.apache.inlong.agent.utils.DebeziumOffsetSerializer;
-import org.apache.inlong.common.metric.MetricRegister;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.inlong.agent.metrics.AgentMetricItem.KEY_INLONG_GROUP_ID;
-import static org.apache.inlong.agent.metrics.AgentMetricItem.KEY_INLONG_STREAM_ID;
-import static org.apache.inlong.agent.metrics.AgentMetricItem.KEY_PLUGIN_ID;
-
-/**
- * Abstract reader, init reader and reader metrics
- */
-public abstract class AbstractReader implements Reader {
-
- protected static final AtomicLong METRIC_INDEX = new AtomicLong(0);
- private static final Logger LOGGER = LoggerFactory.getLogger(AbstractReader.class);
- protected String inlongGroupId;
- protected String inlongStreamId;
- // metric
- protected AgentMetricItemSet metricItemSet;
- protected AgentMetricItem readerMetric;
- protected String metricName;
- protected Map<String, String> dimensions;
-
- @Override
- public void init(InstanceProfile profile) {
- inlongGroupId = profile.getInlongGroupId();
- inlongStreamId = profile.getInlongStreamId();
-
- this.dimensions = new HashMap<>();
- dimensions.put(KEY_PLUGIN_ID, this.getClass().getSimpleName());
- dimensions.put(KEY_INLONG_GROUP_ID, inlongGroupId);
- dimensions.put(KEY_INLONG_STREAM_ID, inlongStreamId);
- metricName = String.join("-", this.getClass().getSimpleName(),
- String.valueOf(METRIC_INDEX.incrementAndGet()));
- this.metricItemSet = new AgentMetricItemSet(metricName);
- MetricRegister.register(metricItemSet);
- readerMetric = metricItemSet.findMetricItem(dimensions);
- }
-
- public String getInlongGroupId() {
- return inlongGroupId;
- }
-
- /**
- * specific offsets
- *
- * @param server specific server
- * @param file specific offset file
- * @param pos specific offset pos
- * @return
- */
- public String serializeOffset(final String server, final String file,
- final String pos) {
- Map<String, Object> sourceOffset = new HashMap<>();
- sourceOffset.put("file", file);
- sourceOffset.put("pos", pos);
- DebeziumOffset specificOffset = new DebeziumOffset();
- specificOffset.setSourceOffset(sourceOffset);
- Map<String, String> sourcePartition = new HashMap<>();
- sourcePartition.put("server", server);
- specificOffset.setSourcePartition(sourcePartition);
- byte[] serializedOffset = new byte[0];
- try {
- serializedOffset = DebeziumOffsetSerializer.INSTANCE.serialize(specificOffset);
- } catch (IOException e) {
- LOGGER.error("serialize offset message error", e);
- }
- return new String(serializedOffset, StandardCharsets.UTF_8);
- }
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/PostgreSQLReaderTest.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/PostgreSQLReaderTest.java
deleted file mode 100644
index 6a85e054f6..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/PostgreSQLReaderTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.SnapshotModeConstants;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.PostgreSQLReader;
-import org.apache.inlong.agent.pojo.DebeziumFormat;
-
-import com.google.gson.Gson;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_GROUP_ID;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_STREAM_ID;
-
-/**
- * Test for PostgreSQL reader
- */
-public class PostgreSQLReaderTest {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PostgreSQLReaderTest.class);
- private static final Gson GSON = new Gson();
-
- @Test
- public void testDebeziumFormat() {
- String debeziumJson = "{\n"
- + " \"before\": null,\n"
- + " \"after\": {\n"
- + " \"id\": 1004,\n"
- + " \"first_name\": \"name1\",\n"
- + " \"last_name\": \"name2\"\n"
- + " },\n"
- + " \"source\": {\n"
- + " \"version\": \"12\",\n"
- + " \"name\": \"myserver\",\n"
- + " \"ts_sec\": 0,\n"
- + " \"gtid\": null,\n"
- + " \"file\": \"000000010000000000000001\",\n"
- + " \"row\": 0,\n"
- + " \"snapshot\": true,\n"
- + " \"thread\": null,\n"
- + " \"db\": \"postgres\",\n"
- + " \"table\": \"customers\"\n"
- + " },\n"
- + " \"op\": \"r\",\n"
- + " \"ts_ms\": 1486500577691\n"
- + " }";
- DebeziumFormat debeziumFormat = GSON.fromJson(debeziumJson, DebeziumFormat.class);
- Assert.assertEquals("customers", debeziumFormat.getSource().getTable());
- Assert.assertEquals("true", debeziumFormat.getSource().getSnapshot());
- }
-
- /**
- * this test is used for testing collect data from postgreSQL in unit test,
- * and it may cause failure in compile
- * thus we annotate it.
- */
- // @Test
- public void postgresLoadTest() {
- InstanceProfile jobProfile = new InstanceProfile();
- jobProfile.set(PostgreSQLReader.JOB_POSTGRESQL_USER, "postgres");
- jobProfile.set(PostgreSQLReader.JOB_DATABASE_SERVER_NAME, "postgres");
- jobProfile.set(PostgreSQLReader.JOB_DATABASE_PLUGIN_NAME, "pgoutput");
- jobProfile.set(PostgreSQLReader.JOB_DATABASE_PASSWORD, "123456");
- jobProfile.set(PostgreSQLReader.JOB_DATABASE_HOSTNAME, "localhost");
- jobProfile.set(PostgreSQLReader.JOB_DATABASE_PORT, "5432");
- jobProfile.set(PostgreSQLReader.JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE, "000000010000000000000001");
- jobProfile.set(PostgreSQLReader.JOB_DATABASE_SNAPSHOT_MODE, SnapshotModeConstants.INITIAL);
- jobProfile.set(PostgreSQLReader.JOB_DATABASE_DBNAME, "postgres");
- jobProfile.set("job.instance.id", "_1");
- jobProfile.set(PROXY_INLONG_GROUP_ID, "groupid");
- jobProfile.set(PROXY_INLONG_STREAM_ID, "streamid");
- PostgreSQLReader postgreSQLReader = new PostgreSQLReader();
- postgreSQLReader.init(jobProfile);
- while (true) {
- Message message = postgreSQLReader.read();
- if (message != null) {
- LOGGER.info("read message is {}", message.toString());
- break;
- }
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/RedisReaderTest.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/RedisReaderTest.java
deleted file mode 100644
index 28407333e8..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/RedisReaderTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.RedisReader;
-
-import org.junit.Ignore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test for redis reader
- */
-public class RedisReaderTest {
-
- private static Logger LOGGER = LoggerFactory.getLogger(RedisReaderTest.class);
-
- /**
- * this test is used for testing collect data from redis in unit test
- * just use in local test
- */
- @Ignore
- public void redisLoadTest() {
- InstanceProfile jobProfile = new InstanceProfile();
- jobProfile.set(RedisReader.JOB_REDIS_HOSTNAME, "localhost");
- jobProfile.set(RedisReader.JOB_REDIS_PORT, "6379");
- jobProfile.set(RedisReader.JOB_REDIS_OFFSET, "7945");
- jobProfile.set(RedisReader.JOB_REDIS_REPLID, "33a016fc98ad27f36218c7648a7a0774a79547d8");
- jobProfile.set("job.instance.id", "_1");
- RedisReader redisReader = new RedisReader();
- redisReader.init(jobProfile);
- while (true) {
- Message message = redisReader.read();
- if (message != null) {
- LOGGER.info("read message is {}", message);
- break;
- }
- }
- }
-
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestBinlogReader.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestBinlogReader.java
deleted file mode 100644
index 6b9b577754..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestBinlogReader.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.JobProfile;
-import org.apache.inlong.agent.constant.SnapshotModeConstants;
-import org.apache.inlong.agent.plugin.sources.reader.BinlogReader;
-import org.apache.inlong.agent.pojo.DebeziumFormat;
-
-import com.google.gson.Gson;
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_GROUP_ID;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_STREAM_ID;
-
-public class TestBinlogReader {
-
- private static Gson gson = new Gson();
-
- @Test
- public void testDebeziumFormat() {
- String debeziumJson = "{\n"
- + " \"before\": null,\n"
- + " \"after\": {\n"
- + " \"id\": 1004,\n"
- + " \"first_name\": \"Anne\",\n"
- + " \"last_name\": \"Kretchmar\",\n"
- + " \"email\": \"annek@noanswer.org\"\n"
- + " },\n"
- + " \"source\": {\n"
- + " \"version\": \"1.8.1.Final\",\n"
- + " \"name\": \"dbserver1\",\n"
- + " \"server_id\": 0,\n"
- + " \"ts_sec\": 0,\n"
- + " \"gtid\": null,\n"
- + " \"file\": \"mysql-bin.000003\",\n"
- + " \"pos\": 154,\n"
- + " \"row\": 0,\n"
- + " \"snapshot\": true,\n"
- + " \"thread\": null,\n"
- + " \"db\": \"inventory\",\n"
- + " \"table\": \"customers\"\n"
- + " },\n"
- + " \"op\": \"r\",\n"
- + " \"ts_ms\": 1486500577691\n"
- + " }";
- DebeziumFormat debeziumFormat = gson
- .fromJson(debeziumJson, DebeziumFormat.class);
- Assert.assertEquals("customers", debeziumFormat.getSource().getTable());
- }
-
- // @Test
- public void binlogStartSpacialTest() throws Exception {
- JobProfile jobProfile = new JobProfile();
- jobProfile.set(BinlogReader.JOB_DATABASE_USER, "root");
- jobProfile.set(BinlogReader.JOB_DATABASE_PASSWORD, "123456");
- jobProfile.set(BinlogReader.JOB_DATABASE_HOSTNAME, "");
- jobProfile.set(BinlogReader.JOB_DATABASE_PORT, "3307");
- jobProfile.set(BinlogReader.JOB_DATABASE_WHITELIST, "etl");
- jobProfile.set(BinlogReader.JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE, "mysql-bin.000001");
- jobProfile.set(BinlogReader.JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS, "5641");
- jobProfile.set(BinlogReader.JOB_DATABASE_SNAPSHOT_MODE, SnapshotModeConstants.SPECIFIC_OFFSETS);
- // jobProfile.set(BinlogReader.JOB_DATABASE_STORE_HISTORY_FILENAME, "");
- jobProfile.set(BinlogReader.JOB_DATABASE_STORE_OFFSET_INTERVAL_MS, String.valueOf(6000L));
- jobProfile.set("job.instance.id", "_1");
- jobProfile.set(PROXY_INLONG_GROUP_ID, "groupid");
- jobProfile.set(PROXY_INLONG_STREAM_ID, "streamid");
- BinlogReader binlogReader = new BinlogReader();
- binlogReader.init(jobProfile);
- while (true) {
-
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestKafkaReader.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestKafkaReader.java
deleted file mode 100644
index f9f1185339..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestKafkaReader.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
-import org.apache.kafka.clients.consumer.MockConsumer;
-import org.apache.kafka.clients.consumer.OffsetResetStrategy;
-import org.apache.kafka.common.TopicPartition;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.nio.charset.StandardCharsets;
-import java.time.Duration;
-import java.util.Collections;
-import java.util.HashMap;
-
-public class TestKafkaReader {
-
- @Test
- public void testKafkaConsumerInit() {
- MockConsumer<String, String> mockConsumer = new MockConsumer<>(OffsetResetStrategy.EARLIEST);
- final String topic = "my_topic";
-
- mockConsumer.assign(Collections.singletonList(new TopicPartition(topic, 0)));
- HashMap<TopicPartition, Long> beginningOffsets = new HashMap<>();
- beginningOffsets.put(new TopicPartition(topic, 0), 0L);
- mockConsumer.updateBeginningOffsets(beginningOffsets);
-
- mockConsumer.addRecord(new ConsumerRecord<>(topic, 0, 0L, "test_key", "test_value"));
- ConsumerRecords<String, String> records = mockConsumer.poll(Duration.ofMillis(1000));
- for (ConsumerRecord<String, String> record : records) {
- byte[] recordValue = record.value().getBytes(StandardCharsets.UTF_8);
- Assert.assertArrayEquals("test_value".getBytes(StandardCharsets.UTF_8), recordValue);
- }
- }
-
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMongoDBReader.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMongoDBReader.java
deleted file mode 100644
index 0c7cd36d10..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMongoDBReader.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.TaskConstants;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.MongoDBReader;
-import org.apache.inlong.agent.pojo.DebeziumFormat;
-
-import com.alibaba.fastjson.JSONPath;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.UUID;
-
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_GROUP_ID;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_STREAM_ID;
-
-/**
- * TestMongoDBReader : TestMongoDBReader
- */
-public class TestMongoDBReader {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TestMongoDBReader.class);
-
- /**
- * Change event format verification
- */
- @Test
- public void testDebeziumFormat() {
- String json = "{\n"
- + " \"payload\": {\n"
- + " \"after\": \"{\\\"_id\\\":{\\\"$oid\\\":\\\"6304d21d96befa25a3630c5e\\\"}"
- + ",\\\"orderId\\\":\\\"o0011\\\",\\\"userId\\\":\\\"u0012\\\",\\\"items\\\":"
- + "[{\\\"itemId2\\\":\\\"i001\\\",\\\"itemName\\\":\\\"yyy\\\"},"
- + "{\\\"itemId\\\":\\\"i002\\\",\\\"itemName\\\":\\\"yyy2\\\"}]}\",\n"
- + " \"patch\": null,\n"
- + " \"filter\": null,\n"
- + " \"updateDescription\": {\n"
- + " \"removedFields\": null,\n"
- + " \"updatedFields\": \"{\\\"items\\\":[{\\\"itemId2\\\":\\\"i001\\\", "
- + "\\\"itemName\\\":\\\"xxx\\\"}, {\\\"itemId\\\":\\\"i002\\\", \\\"itemName\\\":\\\"xxx2\\\"}], "
- + "\\\"userId\\\":\\\"u0012\\\"}\",\n"
- + " \"truncatedArrays\": null\n"
- + " },\n"
- + " \"source\": {\n"
- + " \"version\": \"1.8.0.Final\",\n"
- + " \"connector\": \"mongodb\",\n"
- + " \"name\": \"myrs\",\n"
- + " \"ts_ms\": 1661332000000,\n"
- + " \"snapshot\": \"false\",\n"
- + " \"db\": \"mall\",\n"
- + " \"sequence\": null,\n"
- + " \"rs\": \"myrs\",\n"
- + " \"collection\": \"order\",\n"
- + " \"ord\": 1,\n"
- + " \"h\": null,\n"
- + " \"tord\": null,\n"
- + " \"stxnid\": null,\n"
- + " \"lsid\": null,\n"
- + " \"txnNumber\": null\n"
- + " },\n"
- + " \"op\": \"u\",\n"
- + " \"ts_ms\": 1661332000257,\n"
- + " \"transaction\": null\n"
- + " }\n"
- + "}";
- DebeziumFormat debeziumFormat = JSONPath.read(json, "$.payload", DebeziumFormat.class);
- Assert.assertEquals("order", debeziumFormat.getSource().getCollection());
- Assert.assertEquals("false", debeziumFormat.getSource().getSnapshot());
- }
-
- /**
- * Use local Mongo shard cluster for temporary testing
- */
- @Ignore
- public void readChangeEventFromMongo() {
- InstanceProfile jobProfile = new InstanceProfile();
- jobProfile.set("job.mongoJob.hosts", "localhost:37018");
- jobProfile.set("job.mongoJob.user", "mongo");
- jobProfile.set("job.mongoJob.password", "root");
- jobProfile.set("job.mongoJob.name", "myrs");
- jobProfile.set("job.mongoJob.connectMaxAttempts", "3");
- jobProfile.set("job.mongoJob.databaseIncludeList", "mall");
- jobProfile.set("job.mongoJob.collectionIncludeList", "order");
- jobProfile.set("job.mongoJob.snapshotMode", "never");
- jobProfile.set(TaskConstants.JOB_INSTANCE_ID, UUID.randomUUID().toString());
-
- jobProfile.set(PROXY_INLONG_GROUP_ID, UUID.randomUUID().toString());
- jobProfile.set(PROXY_INLONG_STREAM_ID, UUID.randomUUID().toString());
- MongoDBReader mongoReader = new MongoDBReader();
- mongoReader.init(jobProfile);
- while (true) {
- Message message = mongoReader.read();
- if (message != null) {
- LOGGER.info("event content: {}", message);
- }
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMqttConnect.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMqttConnect.java
deleted file mode 100644
index 8ca9785e6d..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMqttConnect.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.TaskProfile;
-import org.apache.inlong.agent.constant.TaskConstants;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.file.Reader;
-import org.apache.inlong.agent.utils.AgentUtils;
-
-import org.junit.Ignore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.assertNotNull;
-
-public class TestMqttConnect {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TestMqttConnect.class);
-
- /**
- * Just using in local test.
- */
- @Ignore
- public void testMqttReader() throws Exception {
- TaskProfile jobProfile = TaskProfile.parseJsonStr("{}");
- jobProfile.set(TaskConstants.TASK_MQTT_SERVER_URI, "tcp://broker.hivemq.com:1883");
- jobProfile.set(TaskConstants.TASK_MQTT_CLIENT_ID_PREFIX, "mqtt_client");
- jobProfile.set(TaskConstants.TASK_MQTT_USERNAME, "test");
- jobProfile.set(TaskConstants.TASK_MQTT_PASSWORD, "test");
- jobProfile.set(TaskConstants.TASK_MQTT_TOPIC, "testtopic/mqtt/p1/ebr/delivered,testtopic/NARTU2");
- jobProfile.set(TaskConstants.TASK_MQTT_QOS, "0");
- jobProfile.set("job.instance.id", "_1");
-
- final MqttSource source = new MqttSource();
- List<Reader> readers = source.split(jobProfile);
- ExecutorService threadPool = Executors.newFixedThreadPool(5);
- for (Reader reader : readers) {
- threadPool.submit(new Runnable() {
-
- @Override
- public void run() {
- reader.init(jobProfile.createInstanceProfile("", "", jobProfile.getCycleUnit(), "",
- AgentUtils.getCurrentTime()));
- while (!reader.isFinished()) {
- Message message = reader.read();
- if (Objects.nonNull(message)) {
- assertNotNull(message.getBody());
- LOGGER.info("the mqtt reader header: {}, message: {}", message.getHeader(), message);
- }
- }
- }
- });
- }
- TimeUnit.SECONDS.sleep(60);
- }
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMqttReader.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMqttReader.java
deleted file mode 100644
index a2652869ba..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestMqttReader.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.CommonConstants;
-import org.apache.inlong.agent.constant.TaskConstants;
-import org.apache.inlong.agent.message.DefaultMessage;
-import org.apache.inlong.agent.metrics.AgentMetricItem;
-import org.apache.inlong.agent.metrics.AgentMetricItemSet;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.MqttReader;
-import org.apache.inlong.common.metric.MetricRegister;
-
-import org.apache.commons.lang3.StringUtils;
-import org.eclipse.paho.client.mqttv3.MqttClient;
-import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.field;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
-/**
- * Test cases for {@link MqttReader}.
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({MqttReader.class, MetricRegister.class})
-@PowerMockIgnore({"javax.management.*"})
-public class TestMqttReader {
-
- private MqttReader reader;
-
- @Mock
- private InstanceProfile jobProfile;
-
- @Mock
- private AgentMetricItemSet agentMetricItemSet;
-
- @Mock
- private AgentMetricItem agentMetricItem;
-
- @Mock
- private MqttClient mqttClient;
-
- @Mock
- private LinkedBlockingQueue<DefaultMessage> queue;
-
- @Mock
- private DefaultMessage message;
-
- private AtomicLong atomicLong;
-
- private AtomicLong atomicCountLong;
-
- private String topic;
-
- private static final String INSTANCE_ID = "instanceId";
-
- @Before
- public void setUp() throws Exception {
- final String serverURI = "tcp://broker.hivemq.com:1883";
- final String username = "test";
- final String password = "test";
- final String qos = "0";
- final String clientIdPrefix = "mqtt_client";
- final String groupId = "group01";
- final String streamId = "stream01";
-
- atomicLong = new AtomicLong(0L);
- atomicCountLong = new AtomicLong(0L);
-
- topic = "testtopic/mqtt/p1/ebr/delivered";
-
- when(jobProfile.get(eq(CommonConstants.PROXY_INLONG_GROUP_ID), anyString())).thenReturn(groupId);
- when(jobProfile.get(eq(CommonConstants.PROXY_INLONG_STREAM_ID), anyString())).thenReturn(streamId);
- when(jobProfile.get(eq(TaskConstants.TASK_MQTT_USERNAME))).thenReturn(username);
- when(jobProfile.get(eq(TaskConstants.TASK_MQTT_PASSWORD))).thenReturn(password);
- when(jobProfile.get(eq(TaskConstants.TASK_MQTT_SERVER_URI))).thenReturn(serverURI);
- when(jobProfile.get(eq(TaskConstants.TASK_MQTT_QOS))).thenReturn(qos);
- when(jobProfile.get(eq(TaskConstants.TASK_MQTT_CLIENT_ID_PREFIX))).thenReturn(clientIdPrefix);
- when(jobProfile.getInstanceId()).thenReturn(INSTANCE_ID);
- when(jobProfile.getInt(eq(TaskConstants.TASK_MQTT_QUEUE_SIZE), eq(1000))).thenReturn(1000);
-
- // mock MqttClient
- whenNew(MqttClient.class).withArguments(anyString(), anyString(), any(MemoryPersistence.class))
- .thenReturn(mqttClient);
-
- // mock queue
- whenNew(LinkedBlockingQueue.class).withArguments(anyInt()).thenReturn(queue);
- when(queue.poll()).thenReturn(message);
-
- // mock metrics
- whenNew(AgentMetricItemSet.class).withArguments(anyString()).thenReturn(agentMetricItemSet);
- when(agentMetricItemSet.findMetricItem(any())).thenReturn(agentMetricItem);
- field(AgentMetricItem.class, "pluginReadCount").set(agentMetricItem, atomicLong);
- field(AgentMetricItem.class, "pluginReadSuccessCount").set(agentMetricItem, atomicCountLong);
-
- // init method
- mockStatic(MetricRegister.class);
- (reader = new MqttReader(topic)).init(jobProfile);
- }
-
- /**
- * Test cases for {@link MqttReader#read()}.
- */
- @Test
- public void testRead() {
- Message actual = reader.read();
- assertEquals(message, actual);
- }
-
- /**
- * Test cases for {@link MqttReader#destroy()}.
- */
- @Test
- public void testDestroy() throws Exception {
- assertFalse(reader.isFinished());
- reader.destroy();
- verify(mqttClient).disconnect();
- }
-
- /**
- * Test cases for {@link MqttReader#finishRead()}.
- */
- @Test
- public void testFinishRead() {
- assertFalse(reader.isFinished());
- reader.finishRead();
- assertTrue(reader.isFinished());
- }
-
- /**
- * Test cases for {@link MqttReader#isSourceExist()}.
- */
- @Test
- public void testIsSourceExist() {
- assertTrue(reader.isSourceExist());
- }
-
- /**
- * Test cases for {@link MqttReader#getSnapshot()}.
- */
- @Test
- public void testGetSnapshot() {
- assertEquals(StringUtils.EMPTY, reader.getSnapshot());
- }
-
- /**
- * Test cases for {@link MqttReader#getReadSource()}.
- */
- @Test
- public void testGetReadSource() {
- assertEquals(INSTANCE_ID, reader.getReadSource());
- }
-
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestOracleConnect.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestOracleConnect.java
deleted file mode 100644
index 7f2e9e3c81..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestOracleConnect.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.TaskConstants;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.OracleReader;
-
-import org.junit.Ignore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.UUID;
-
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_GROUP_ID;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_STREAM_ID;
-
-/**
- * Test cases for {@link OracleReader}.
- */
-public class TestOracleConnect {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TestOracleConnect.class);
-
- @Ignore
- public void testOracle() {
- InstanceProfile jobProfile = new InstanceProfile();
- jobProfile.set("job.oracleJob.hostname", "192.168.101.11");
- jobProfile.set("job.oracleJob.port", "49161");
- jobProfile.set("job.oracleJob.user", "c##admin");
- jobProfile.set("job.oracleJob.password", "inlong");
- jobProfile.set("job.oracleJob.sid", "xe");
- jobProfile.set("job.oracleJob.dbname", "xe");
- jobProfile.set("job.oracleJob.serverName", "server1");
- jobProfile.set("instance.id", "instance_test");
- jobProfile.set(TaskConstants.JOB_INSTANCE_ID, UUID.randomUUID().toString());
- jobProfile.set(PROXY_INLONG_GROUP_ID, UUID.randomUUID().toString());
- jobProfile.set(PROXY_INLONG_STREAM_ID, UUID.randomUUID().toString());
- OracleReader oracleReader = new OracleReader();
- oracleReader.init(jobProfile);
- while (true) {
- Message message = oracleReader.read();
- if (message != null) {
- LOGGER.info("event content: {}", message);
- }
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestOracleReader.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestOracleReader.java
deleted file mode 100644
index 7e16d0b5da..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestOracleReader.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.CommonConstants;
-import org.apache.inlong.agent.constant.OracleConstants;
-import org.apache.inlong.agent.metrics.AgentMetricItem;
-import org.apache.inlong.agent.metrics.AgentMetricItemSet;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.OracleReader;
-import org.apache.inlong.agent.plugin.sources.snapshot.OracleSnapshotBase;
-import org.apache.inlong.common.metric.MetricRegister;
-
-import io.debezium.engine.ChangeEvent;
-import io.debezium.engine.DebeziumEngine;
-import org.apache.commons.lang3.tuple.Pair;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.field;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
-/**
- * Test cases for {@link OracleReader}.
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({DebeziumEngine.class, Executors.class, MetricRegister.class, OracleReader.class})
-@PowerMockIgnore({"javax.management.*"})
-public class TestOracleReader {
-
- private OracleReader reader;
-
- @Mock
- private InstanceProfile jobProfile;
-
- @Mock
- private AgentMetricItemSet agentMetricItemSet;
-
- @Mock
- private AgentMetricItem agentMetricItem;
-
- @Mock
- private OracleSnapshotBase oracleSnapshot;
-
- @Mock
- private DebeziumEngine.Builder builder;
-
- @Mock
- private ExecutorService executorService;
-
- @Mock
- private LinkedBlockingQueue<Pair<String, String>> oracleMessageQueue;
-
- @Mock
- private DebeziumEngine<ChangeEvent<String, String>> engine;
-
- private AtomicLong atomicLong;
-
- private AtomicLong atomicCountLong;
-
- private final String instanceId = "s4bc475560b4444dbd4e9812ab1fd64d";
-
- @Before
- public void setUp() throws Exception {
- final String username = "sa";
- final String password = "123456";
- final String hostname = "127.0.0.1";
- final String port = "1434";
- final String groupId = "group01";
- final String streamId = "stream01";
- final String dbName = "testdb";
- final String serverName = "serverName";
- final String offsetFlushIntervalMs = "1000";
- final String offsetStoreFileName = "/opt/offset.dat";
- final String snapshotMode = OracleConstants.INITIAL;
- final int queueSize = 1000;
- final String databaseStoreHistoryName = "/opt/history.dat";
- final String offset = "111";
- final String specificOffsetFile = "";
- final String specificOffsetPos = "-1";
-
- atomicLong = new AtomicLong(0L);
- atomicCountLong = new AtomicLong(0L);
-
- when(jobProfile.getInstanceId()).thenReturn(instanceId);
- when(jobProfile.get(eq(CommonConstants.PROXY_INLONG_GROUP_ID), anyString())).thenReturn(groupId);
- when(jobProfile.get(eq(CommonConstants.PROXY_INLONG_STREAM_ID), anyString())).thenReturn(streamId);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_USER))).thenReturn(username);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_PASSWORD))).thenReturn(password);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_HOSTNAME))).thenReturn(hostname);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_PORT))).thenReturn(port);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_DBNAME))).thenReturn(dbName);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_SERVER_NAME))).thenReturn(serverName);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_STORE_OFFSET_INTERVAL_MS), anyString())).thenReturn(
- offsetFlushIntervalMs);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_STORE_HISTORY_FILENAME), anyString())).thenReturn(
- offsetStoreFileName);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_SNAPSHOT_MODE), anyString())).thenReturn(snapshotMode);
- when(jobProfile.getInt(eq(OracleReader.JOB_DATABASE_QUEUE_SIZE), anyInt())).thenReturn(queueSize);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_STORE_HISTORY_FILENAME))).thenReturn(databaseStoreHistoryName);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_OFFSETS), anyString())).thenReturn(offset);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE), anyString())).thenReturn(
- specificOffsetFile);
- when(jobProfile.get(eq(OracleReader.JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS), anyString())).thenReturn(
- specificOffsetPos);
- whenNew(OracleSnapshotBase.class).withAnyArguments().thenReturn(oracleSnapshot);
-
- // mock oracleMessageQueue
- whenNew(LinkedBlockingQueue.class).withAnyArguments().thenReturn(oracleMessageQueue);
-
- // mock DebeziumEngine
- mockStatic(DebeziumEngine.class);
- when(DebeziumEngine.create(io.debezium.engine.format.Json.class)).thenReturn(builder);
- when(builder.using(any(Properties.class))).thenReturn(builder);
- when(builder.notifying(any(DebeziumEngine.ChangeConsumer.class))).thenReturn(builder);
- when(builder.using(any(DebeziumEngine.CompletionCallback.class))).thenReturn(builder);
- when(builder.build()).thenReturn(engine);
-
- // mock executorService
- mockStatic(Executors.class);
- when(Executors.newSingleThreadExecutor()).thenReturn(executorService);
-
- // mock metrics
- whenNew(AgentMetricItemSet.class).withArguments(anyString()).thenReturn(agentMetricItemSet);
- when(agentMetricItemSet.findMetricItem(any())).thenReturn(agentMetricItem);
- field(AgentMetricItem.class, "pluginReadCount").set(agentMetricItem, atomicLong);
- field(AgentMetricItem.class, "pluginReadSuccessCount").set(agentMetricItem, atomicCountLong);
-
- // init method
- mockStatic(MetricRegister.class);
- (reader = new OracleReader()).init(jobProfile);
- }
-
- /**
- * Test cases for {@link OracleReader#read()}.
- */
- @Test
- public void testRead() throws Exception {
- final String right = "value";
- final String left = "key";
- final String dataKey = "dataKey";
- when(oracleMessageQueue.isEmpty()).thenReturn(true);
- assertEquals(null, reader.read());
- when(oracleMessageQueue.isEmpty()).thenReturn(false);
- when(oracleMessageQueue.poll()).thenReturn(Pair.of(left, right));
- Message result = reader.read();
- assertEquals(String.join(right, "\"", "\""), result.toString());
- assertEquals(left, result.getHeader().get(dataKey));
- }
-
- /**
- * Test cases for {@link OracleReader#destroy()}.
- */
- @Test
- public void testDestroy() throws Exception {
- assertFalse(reader.isDestroyed());
- reader.destroy();
- verify(executorService).shutdownNow();
- verify(oracleSnapshot).close();
- assertTrue(reader.isDestroyed());
- }
-
- /**
- * Test cases for {@link OracleReader#finishRead()}.
- */
- @Test
- public void testFinishRead() throws Exception {
- assertFalse(reader.isFinished());
- reader.finishRead();
- assertTrue(reader.isFinished());
- }
-
- /**
- * Test cases for {@link OracleReader#isSourceExist()}.
- */
- @Test
- public void testIsSourceExist() {
- assertTrue(reader.isSourceExist());
- }
-
- /**
- * Test cases for {@link OracleReader#getSnapshot()}.
- */
- @Test
- public void testGetSnapshot() {
- final String snapShort = "snapShort";
- when(oracleSnapshot.getSnapshot()).thenReturn(snapShort);
- assertEquals(snapShort, reader.getSnapshot());
- }
-
- /**
- * Test cases for {@link OracleReader#getReadSource()}.
- */
- @Test
- public void testGetReadSource() {
- assertEquals(instanceId, reader.getReadSource());
- }
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestSQLServerConnect.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestSQLServerConnect.java
deleted file mode 100644
index b97b1d5698..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestSQLServerConnect.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.TaskConstants;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.SQLServerReader;
-
-import org.junit.Ignore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.UUID;
-
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_GROUP_ID;
-import static org.apache.inlong.agent.constant.CommonConstants.PROXY_INLONG_STREAM_ID;
-
-/**
- * Test cases for {@link SQLServerReader}.
- */
-public class TestSQLServerConnect {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(TestSQLServerConnect.class);
-
- /**
- * Just using in local test.
- */
-
- @Ignore
- public void testSqlServer() {
- InstanceProfile jobProfile = new InstanceProfile();
- jobProfile.set("job.sqlserverJob.hostname", "localhost");
- jobProfile.set("job.sqlserverJob.port", "1434");
- jobProfile.set("job.sqlserverJob.user", "sa");
- jobProfile.set("job.sqlserverJob.password", "123456");
- jobProfile.set("job.sqlserverJob.dbname", "inlong");
- jobProfile.set("job.sqlserverJob.serverName", "fullfillment");
- jobProfile.set(TaskConstants.JOB_INSTANCE_ID, UUID.randomUUID().toString());
- jobProfile.set(PROXY_INLONG_GROUP_ID, UUID.randomUUID().toString());
- jobProfile.set(PROXY_INLONG_STREAM_ID, UUID.randomUUID().toString());
- SQLServerReader sqlServerReader = new SQLServerReader();
- sqlServerReader.init(jobProfile);
- while (true) {
- Message message = sqlServerReader.read();
- if (message != null) {
- LOGGER.info("event content: {}", message);
- }
- }
- }
-}
diff --git a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestSQLServerReader.java b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestSQLServerReader.java
deleted file mode 100644
index ae53fbb5e1..0000000000
--- a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestSQLServerReader.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.plugin.sources;
-
-import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.CommonConstants;
-import org.apache.inlong.agent.constant.SqlServerConstants;
-import org.apache.inlong.agent.metrics.AgentMetricItem;
-import org.apache.inlong.agent.metrics.AgentMetricItemSet;
-import org.apache.inlong.agent.plugin.Message;
-import org.apache.inlong.agent.plugin.sources.reader.SQLServerReader;
-import org.apache.inlong.agent.plugin.sources.snapshot.SqlServerSnapshotBase;
-import org.apache.inlong.common.metric.MetricRegister;
-
-import io.debezium.engine.ChangeEvent;
-import io.debezium.engine.DebeziumEngine;
-import org.apache.commons.lang3.tuple.Pair;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import java.util.Properties;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.Mockito.verify;
-import static org.powermock.api.mockito.PowerMockito.field;
-import static org.powermock.api.mockito.PowerMockito.mockStatic;
-import static org.powermock.api.mockito.PowerMockito.when;
-import static org.powermock.api.mockito.PowerMockito.whenNew;
-
-/**
- * Test cases for {@link SQLServerReader}.
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({DebeziumEngine.class, Executors.class, MetricRegister.class, SQLServerReader.class})
-@PowerMockIgnore({"javax.management.*"})
-public class TestSQLServerReader {
-
- private SQLServerReader reader;
-
- @Mock
- private InstanceProfile jobProfile;
-
- @Mock
- private AgentMetricItemSet agentMetricItemSet;
-
- @Mock
- private AgentMetricItem agentMetricItem;
-
- @Mock
- private SqlServerSnapshotBase sqlServerSnapshot;
-
- @Mock
- private DebeziumEngine.Builder builder;
-
- @Mock
- private ExecutorService executorService;
-
- @Mock
- private LinkedBlockingQueue<Pair<String, String>> sqlServerMessageQueue;
-
- @Mock
- private DebeziumEngine<ChangeEvent<String, String>> engine;
-
- private AtomicLong atomicLong;
-
- private AtomicLong atomicCountLong;
-
- private final String instanceId = "s4bc475560b4444dbd4e9812ab1fd64d";
-
- @Before
- public void setUp() throws Exception {
- final String username = "sa";
- final String password = "123456";
- final String hostname = "127.0.0.1";
- final String port = "1434";
- final String groupId = "group01";
- final String streamId = "stream01";
- final String dbName = "inlong";
- final String serverName = "server1";
- final String offsetFlushIntervalMs = "1000";
- final String offsetStoreFileName = "/opt/offset.dat";
- final String snapshotMode = SqlServerConstants.INITIAL;
- final int queueSize = 1000;
- final String databaseStoreHistoryName = "/opt/history.dat";
- final String offset = "111";
- final String specificOffsetFile = "";
- final String specificOffsetPos = "-1";
-
- atomicLong = new AtomicLong(0L);
- atomicCountLong = new AtomicLong(0L);
-
- when(jobProfile.getInstanceId()).thenReturn(instanceId);
- when(jobProfile.get(eq(CommonConstants.PROXY_INLONG_GROUP_ID), anyString())).thenReturn(groupId);
- when(jobProfile.get(eq(CommonConstants.PROXY_INLONG_STREAM_ID), anyString())).thenReturn(streamId);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_USER))).thenReturn(username);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_PASSWORD))).thenReturn(password);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_HOSTNAME))).thenReturn(hostname);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_PORT))).thenReturn(port);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_DBNAME))).thenReturn(dbName);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_SERVER_NAME))).thenReturn(serverName);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_STORE_OFFSET_INTERVAL_MS), anyString())).thenReturn(
- offsetFlushIntervalMs);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_STORE_HISTORY_FILENAME), anyString())).thenReturn(
- offsetStoreFileName);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_SNAPSHOT_MODE), anyString())).thenReturn(snapshotMode);
- when(jobProfile.getInt(eq(SQLServerReader.JOB_DATABASE_QUEUE_SIZE), anyInt())).thenReturn(queueSize);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_STORE_HISTORY_FILENAME))).thenReturn(
- databaseStoreHistoryName);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_OFFSETS), anyString())).thenReturn(offset);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_FILE), anyString())).thenReturn(
- specificOffsetFile);
- when(jobProfile.get(eq(SQLServerReader.JOB_DATABASE_OFFSET_SPECIFIC_OFFSET_POS), anyString())).thenReturn(
- specificOffsetPos);
- whenNew(SqlServerSnapshotBase.class).withAnyArguments().thenReturn(sqlServerSnapshot);
-
- // mock sqlServerMessageQueue
- whenNew(LinkedBlockingQueue.class).withAnyArguments().thenReturn(sqlServerMessageQueue);
-
- // mock DebeziumEngine
- mockStatic(DebeziumEngine.class);
- when(DebeziumEngine.create(io.debezium.engine.format.Json.class)).thenReturn(builder);
- when(builder.using(any(Properties.class))).thenReturn(builder);
- when(builder.notifying(any(DebeziumEngine.ChangeConsumer.class))).thenReturn(builder);
- when(builder.using(any(DebeziumEngine.CompletionCallback.class))).thenReturn(builder);
- when(builder.build()).thenReturn(engine);
-
- // mock executorService
- mockStatic(Executors.class);
- when(Executors.newSingleThreadExecutor()).thenReturn(executorService);
-
- // mock metrics
- whenNew(AgentMetricItemSet.class).withArguments(anyString()).thenReturn(agentMetricItemSet);
- when(agentMetricItemSet.findMetricItem(any())).thenReturn(agentMetricItem);
- field(AgentMetricItem.class, "pluginReadCount").set(agentMetricItem, atomicLong);
- field(AgentMetricItem.class, "pluginReadSuccessCount").set(agentMetricItem, atomicCountLong);
-
- // init method
- mockStatic(MetricRegister.class);
- (reader = new SQLServerReader()).init(jobProfile);
- }
-
- /**
- * Test cases for {@link SQLServerReader#read()}.
- */
- @Test
- public void testRead() throws Exception {
- final String right = "value";
- final String left = "key";
- final String dataKey = "dataKey";
- when(sqlServerMessageQueue.isEmpty()).thenReturn(true);
- assertEquals(null, reader.read());
- when(sqlServerMessageQueue.isEmpty()).thenReturn(false);
- when(sqlServerMessageQueue.poll()).thenReturn(Pair.of(left, right));
- Message result = reader.read();
- assertEquals(String.join(right, "\"", "\""), result.toString());
- assertEquals(left, result.getHeader().get(dataKey));
- }
-
- /**
- * Test cases for {@link SQLServerReader#destroy()}.
- */
- @Test
- public void testDestroy() throws Exception {
- assertFalse(reader.isDestroyed());
- reader.destroy();
- verify(executorService).shutdownNow();
- verify(sqlServerSnapshot).close();
- assertTrue(reader.isDestroyed());
- }
-
- /**
- * Test cases for {@link SQLServerReader#finishRead()}.
- */
- @Test
- public void testFinishRead() throws Exception {
- assertFalse(reader.isFinished());
- reader.finishRead();
- assertTrue(reader.isFinished());
- }
-
- /**
- * Test cases for {@link SQLServerReader#isSourceExist()}.
- */
- @Test
- public void testIsSourceExist() {
- assertTrue(reader.isSourceExist());
- }
-
- /**
- * Test cases for {@link SQLServerReader#getSnapshot()}.
- */
- @Test
- public void testGetSnapshot() {
- final String snapShort = "snapShort";
- when(sqlServerSnapshot.getSnapshot()).thenReturn(snapShort);
- assertEquals(snapShort, reader.getSnapshot());
- }
-
- /**
- * Test cases for {@link SQLServerReader#getReadSource()}.
- */
- @Test
- public void testGetReadSource() {
- assertEquals(instanceId, reader.getReadSource());
- }
-}