You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/03/30 06:24:30 UTC
[inlong] branch master updated (ecff1b430 -> d3e76c69f)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from ecff1b430 [INLONG-7697][Sort] Reduce the memory usage of JM when split table chunks (#7726)
add d3e76c69f [INLONG-7635][Sort] MongoDB CDC supports metrics with enable incremental snapshots (#7638)
No new revisions were added by this update.
Summary of changes:
.../debezium/DebeziumDeserializationSchema.java | 7 +
.../table/RowDataDebeziumDeserializeSchema.java | 30 +-
.../sort/cdc/mongodb/DebeziumSourceFunction.java | 2 +-
.../inlong/sort/cdc/mongodb/MongoDBSource.java | 2 +-
.../debezium/DebeziumDeserializationSchema.java | 46 ---
.../debezium/internal/DebeziumChangeFetcher.java | 2 +-
.../MongoDBConnectorDeserializationSchema.java | 2 +-
.../sort/cdc/mongodb/source/IncrementalSource.java | 215 ------------
.../sort/cdc/mongodb/source/MongoDBSource.java | 19 +-
.../cdc/mongodb/source/MongoDBSourceBuilder.java | 22 +-
.../assigners/splitters/MongoDBChunkSplitter.java} | 28 +-
.../splitters/SampleBucketSplitStrategy.java | 169 ++++++++++
.../assigners/splitters/ShardedSplitStrategy.java | 133 ++++++++
.../assigners/splitters/SingleSplitStrategy.java | 67 ++++
.../source/assigners/splitters/SplitContext.java | 95 ++++++
.../source/assigners/splitters/SplitStrategy.java | 56 ++++
.../splitters/SplitVectorSplitStrategy.java | 140 ++++++++
.../mongodb/source/config/MongoDBSourceConfig.java | 228 +++++++++++++
.../source/config/MongoDBSourceConfigFactory.java | 230 +++++++++++++
.../source/config/MongoDBSourceOptions.java | 138 ++++++++
.../mongodb/source/connection/MongoClientPool.java | 58 ++++
.../cdc/mongodb/source/dialect/MongoDBDialect.java | 172 ++++++++++
.../source/offset/ChangeStreamDescriptor.java | 140 ++++++++
.../mongodb/source/offset/ChangeStreamOffset.java | 105 ++++++
.../source/offset/ChangeStreamOffsetFactory.java | 56 ++++
.../reader/IncrementalSourceRecordEmitter.java | 177 ----------
.../source/reader/MongoDBRecordEmitter.java | 54 ++-
.../reader/fetch/MongoDBFetchTaskContext.java | 193 +++++++++++
.../source/reader/fetch/MongoDBScanFetchTask.java | 264 +++++++++++++++
.../reader/fetch/MongoDBStreamFetchTask.java | 352 ++++++++++++++++++++
.../sort/cdc/mongodb/source/utils/MongoUtils.java | 370 +++++++++++++++++++++
.../sort/cdc/mongodb/table/MongoDBTableSource.java | 8 +-
.../mongodb/table/MongoDBTableSourceFactory.java | 11 +-
.../table/RowDataDebeziumDeserializeSchema.java | 2 +-
licenses/inlong-sort-connectors/LICENSE | 24 +-
35 files changed, 3131 insertions(+), 486 deletions(-)
delete mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/debezium/DebeziumDeserializationSchema.java
delete mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/IncrementalSource.java
copy inlong-sort/sort-connectors/{cdc-base/src/main/java/org/apache/inlong/sort/cdc/base/source/assigner/splitter/ChunkSplitter.java => mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/assigners/splitters/MongoDBChunkSplitter.java} (50%)
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/assigners/splitters/SampleBucketSplitStrategy.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/assigners/splitters/ShardedSplitStrategy.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/assigners/splitters/SingleSplitStrategy.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/assigners/splitters/SplitContext.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/assigners/splitters/SplitStrategy.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/assigners/splitters/SplitVectorSplitStrategy.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/config/MongoDBSourceConfig.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/config/MongoDBSourceConfigFactory.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/config/MongoDBSourceOptions.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/connection/MongoClientPool.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/dialect/MongoDBDialect.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/offset/ChangeStreamDescriptor.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/offset/ChangeStreamOffset.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/offset/ChangeStreamOffsetFactory.java
delete mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/reader/IncrementalSourceRecordEmitter.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/reader/fetch/MongoDBFetchTaskContext.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/reader/fetch/MongoDBScanFetchTask.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/reader/fetch/MongoDBStreamFetchTask.java
create mode 100644 inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/source/utils/MongoUtils.java