You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/09/21 02:10:20 UTC

[inlong] branch master updated: [INLONG-8953][Sort] Fix IcebergSource defaults the StartSnapshot to the latest (#8954)

This is an automated email from the ASF dual-hosted git repository.

dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
     new cc569c9999 [INLONG-8953][Sort] Fix IcebergSource defaults the StartSnapshot to the latest (#8954)
cc569c9999 is described below

commit cc569c9999ca77af9e3001eb40d53ff87cb3e49c
Author: vernedeng <ve...@apache.org>
AuthorDate: Thu Sep 21 10:10:14 2023 +0800

    [INLONG-8953][Sort] Fix IcebergSource defaults the StartSnapshot to the latest (#8954)
---
 .../apache/inlong/sort/protocol/constant/IcebergConstant.java  | 10 ++++++++++
 .../inlong/sort/protocol/node/extract/IcebergExtractNode.java  |  5 +++++
 2 files changed, 15 insertions(+)

diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
index 676f7f4435..2cce35fb9b 100644
--- a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
@@ -34,6 +34,7 @@ public class IcebergConstant {
     public static final String WAREHOUSE_KEY = "warehouse";
     public static final String START_SNAPSHOT_ID = "start-snapshot-id";
     public static final String STREAMING = "streaming";
+    public static final String STARTING_STRATEGY_KEY = "starting-strategy";
 
     /**
      * Iceberg supported catalog type
@@ -65,4 +66,13 @@ public class IcebergConstant {
             throw new IllegalArgumentException(String.format("Unsupport catalogType:%s", name));
         }
     }
+
+    public enum StreamingStartingStrategy {
+        TABLE_SCAN_THEN_INCREMENTAL,
+        INCREMENTAL_FROM_LATEST_SNAPSHOT,
+        INCREMENTAL_FROM_EARLIEST_SNAPSHOT,
+        INCREMENTAL_FROM_SNAPSHOT_ID,
+        INCREMENTAL_FROM_SNAPSHOT_TIMESTAMP;
+
+    }
 }
diff --git a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
index 14fec78da8..c48899b887 100644
--- a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
+++ b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
@@ -110,7 +110,10 @@ public class IcebergExtractNode extends ExtractNode implements Serializable {
         options.put(IcebergConstant.TABLE_KEY, tableName);
         options.put(IcebergConstant.CATALOG_TYPE_KEY, catalogType.name());
         options.put(IcebergConstant.CATALOG_NAME_KEY, catalogName);
+        // support streaming only
         options.put(IcebergConstant.STREAMING, "true");
+        options.put(IcebergConstant.STARTING_STRATEGY_KEY,
+                IcebergConstant.StreamingStartingStrategy.INCREMENTAL_FROM_EARLIEST_SNAPSHOT.name());
         if (null != uri) {
             options.put(IcebergConstant.URI_KEY, uri);
         }
@@ -119,6 +122,8 @@ public class IcebergExtractNode extends ExtractNode implements Serializable {
         }
         if (null != startSnapShotId) {
             options.put(IcebergConstant.START_SNAPSHOT_ID, startSnapShotId.toString());
+            options.put(IcebergConstant.STARTING_STRATEGY_KEY,
+                    IcebergConstant.StreamingStartingStrategy.INCREMENTAL_FROM_SNAPSHOT_ID.name());
         }
         return options;
     }