You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/09/21 02:10:20 UTC
[inlong] branch master updated: [INLONG-8953][Sort] Fix IcebergSource defaults the StartSnapshot to the latest (#8954)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new cc569c9999 [INLONG-8953][Sort] Fix IcebergSource defaults the StartSnapshot to the latest (#8954)
cc569c9999 is described below
commit cc569c9999ca77af9e3001eb40d53ff87cb3e49c
Author: vernedeng <ve...@apache.org>
AuthorDate: Thu Sep 21 10:10:14 2023 +0800
[INLONG-8953][Sort] Fix IcebergSource defaults the StartSnapshot to the latest (#8954)
---
.../apache/inlong/sort/protocol/constant/IcebergConstant.java | 10 ++++++++++
.../inlong/sort/protocol/node/extract/IcebergExtractNode.java | 5 +++++
2 files changed, 15 insertions(+)
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
index 676f7f4435..2cce35fb9b 100644
--- a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
@@ -34,6 +34,7 @@ public class IcebergConstant {
public static final String WAREHOUSE_KEY = "warehouse";
public static final String START_SNAPSHOT_ID = "start-snapshot-id";
public static final String STREAMING = "streaming";
+ public static final String STARTING_STRATEGY_KEY = "starting-strategy";
/**
* Iceberg supported catalog type
@@ -65,4 +66,13 @@ public class IcebergConstant {
throw new IllegalArgumentException(String.format("Unsupport catalogType:%s", name));
}
}
+
+ public enum StreamingStartingStrategy {
+ TABLE_SCAN_THEN_INCREMENTAL,
+ INCREMENTAL_FROM_LATEST_SNAPSHOT,
+ INCREMENTAL_FROM_EARLIEST_SNAPSHOT,
+ INCREMENTAL_FROM_SNAPSHOT_ID,
+ INCREMENTAL_FROM_SNAPSHOT_TIMESTAMP;
+
+ }
}
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
index 14fec78da8..c48899b887 100644
--- a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
@@ -110,7 +110,10 @@ public class IcebergExtractNode extends ExtractNode implements Serializable {
options.put(IcebergConstant.TABLE_KEY, tableName);
options.put(IcebergConstant.CATALOG_TYPE_KEY, catalogType.name());
options.put(IcebergConstant.CATALOG_NAME_KEY, catalogName);
+ // support streaming only
options.put(IcebergConstant.STREAMING, "true");
+ options.put(IcebergConstant.STARTING_STRATEGY_KEY,
+ IcebergConstant.StreamingStartingStrategy.INCREMENTAL_FROM_EARLIEST_SNAPSHOT.name());
if (null != uri) {
options.put(IcebergConstant.URI_KEY, uri);
}
@@ -119,6 +122,8 @@ public class IcebergExtractNode extends ExtractNode implements Serializable {
}
if (null != startSnapShotId) {
options.put(IcebergConstant.START_SNAPSHOT_ID, startSnapShotId.toString());
+ options.put(IcebergConstant.STARTING_STRATEGY_KEY,
+ IcebergConstant.StreamingStartingStrategy.INCREMENTAL_FROM_SNAPSHOT_ID.name());
}
return options;
}