You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/06/21 08:17:35 UTC

[incubator-paimon] branch master updated: [core] Introduce filterAndCommit method for StreamTableCommit, which merges filterCommitted and commit (#1414)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 3a91b6be2 [core] Introduce filterAndCommit method for StreamTableCommit, which merges filterCommitted and commit (#1414)
3a91b6be2 is described below

commit 3a91b6be2a755ba9b8a06ca2952c048676603aa2
Author: tsreaper <ts...@gmail.com>
AuthorDate: Wed Jun 21 16:17:31 2023 +0800

    [core] Introduce filterAndCommit method for StreamTableCommit, which merges filterCommitted and commit (#1414)
---
 docs/content/api/java-api.md                       | 12 +++--
 .../paimon/table/sink/StreamTableCommit.java       | 25 ++++++++++
 .../paimon/table/sink/StreamWriteBuilder.java      |  4 +-
 .../apache/paimon/table/sink/TableCommitImpl.java  | 54 ++++++++++++++++------
 .../org/apache/paimon/flink/sink/Committer.java    | 10 ++--
 .../RestoreAndFailCommittableStateManager.java     |  5 +-
 .../apache/paimon/flink/sink/StoreCommitter.java   | 20 ++------
 .../paimon/flink/sink/StoreMultiCommitter.java     | 45 ++++++------------
 8 files changed, 103 insertions(+), 72 deletions(-)

diff --git a/docs/content/api/java-api.md b/docs/content/api/java-api.md
index 48c324f5e..9ddfcc0f4 100644
--- a/docs/content/api/java-api.md
+++ b/docs/content/api/java-api.md
@@ -560,7 +560,7 @@ Key points to achieve exactly-once consistency:
 - Different applications need to use different commitUsers.
 - The commitIdentifier of `StreamTableWrite` and `StreamTableCommit` needs to be consistent, and the
   id needs to be incremented for the next committing.
-- When a failure occurs, if you still have uncommitted `CommitMessage`s, please use `StreamTableCommit#filterCommitted`
+- When a failure occurs, if you still have uncommitted `CommitMessage`s, please use `StreamTableCommit#filterAndCommit`
   to exclude the committed messages by commitIdentifier.
 
 ```java
@@ -594,8 +594,14 @@ public class StreamWriteTable {
         StreamTableCommit commit = writeBuilder.newCommit();
         commit.commit(commitIdentifier, messages);
 
-        // 4. When failover, you can use 'filterCommitted' to filter committed commits.
-        commit.filterCommitted(committedIdentifiers);
+        // 4. When failure occurs and you're not sure if the commit process is successful,
+        //    you can use `filterAndCommit` to retry the commit process.
+        //    Succeeded commits will be automatically skipped.
+        /*
+        Map<Long, List<CommitMessage>> commitIdentifiersAndMessages = new HashMap<>();
+        commitIdentifiersAndMessages.put(commitIdentifier, messages);
+        commit.filterAndCommit(commitIdentifiersAndMessages);
+        */
     }
 }
 ```
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamTableCommit.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamTableCommit.java
index 6a4dc4a25..d2c91dc40 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamTableCommit.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamTableCommit.java
@@ -21,6 +21,7 @@ package org.apache.paimon.table.sink;
 import org.apache.paimon.annotation.Public;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 /**
@@ -35,7 +36,12 @@ public interface StreamTableCommit extends TableCommit {
     /**
      * Filter committed commits. Return uncommitted identifiers. This method is used for failover
      * cases.
+     *
+     * @deprecated Use {@link StreamTableCommit#filterAndCommit} to filter and commit all {@link
+     *     CommitMessage} in question with one method call, instead of calling this method first and
+     *     then call {@link StreamTableCommit#commit}.
      */
+    @Deprecated
     Set<Long> filterCommitted(Set<Long> commitIdentifiers);
 
     /**
@@ -55,10 +61,29 @@ public interface StreamTableCommit extends TableCommit {
      * The check frequency is controlled by 'partition.expiration-check-interval'. Partition
      * expiration will create an 'OVERWRITE' snapshot.
      *
+     * <p>Compared to {@link StreamTableCommit#filterAndCommit}, this method does not check if
+     * {@code commitIdentifier} has been committed, so this method might be faster. Please only use
+     * this method if you can make sure that the {@code commitIdentifier} hasn't been committed
+     * before.
+     *
      * @param commitIdentifier Committed transaction ID, can start from 0. If there are multiple
      *     commits, please increment this ID.
      * @param commitMessages commit messages from table write
      * @see StreamTableWrite#prepareCommit
      */
     void commit(long commitIdentifier, List<CommitMessage> commitMessages);
+
+    /**
+     * Filter out all {@code List<CommitMessage>} which have been committed and commit the remaining
+     * ones.
+     *
+     * <p>Compared to {@link StreamTableCommit#commit}, this method will first check if a {@code
+     * commitIdentifier} has been committed, so this method might be slower. A common usage of this
+     * method is to retry the commit process after a failure.
+     *
+     * @param commitIdentifiersAndMessages a map containing all {@link CommitMessage}s in question.
+     *     The key is the {@code commitIdentifier}.
+     * @return number of {@code List<CommitMessage>} committed.
+     */
+    int filterAndCommit(Map<Long, List<CommitMessage>> commitIdentifiersAndMessages);
 }
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamWriteBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamWriteBuilder.java
index c5465de2d..7f272e075 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamWriteBuilder.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamWriteBuilder.java
@@ -32,8 +32,8 @@ import org.apache.paimon.data.InternalRow;
  *   <li>Different applications need to use different commitUsers.
  *   <li>The commitIdentifier of {@link StreamTableWrite} and {@link StreamTableCommit} needs to be
  *       consistent, and the id needs to be incremented for the next committing.
- *   <li>When a failure occurs, if you still have uncommitted {@link CommitMessage}s, please use
- *       {@link StreamTableCommit#filterCommitted} to exclude the committed messages by
+ *   <li>When a failure occurs, if you still have uncommitted {@link CommitMessage}s, please retry
+ *       with {@link StreamTableCommit#filterAndCommit} to exclude the committed messages by
  *       commitIdentifier.
  * </ul>
  *
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
index 96913e1fc..a25823a0d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
@@ -30,10 +30,12 @@ import javax.annotation.Nullable;
 import java.time.Duration;
 import java.time.LocalDateTime;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.apache.paimon.utils.Preconditions.checkState;
 
@@ -96,18 +98,33 @@ public class TableCommitImpl implements InnerTableCommit {
         return commit.filterCommitted(commitIdentifiers);
     }
 
+    @Override
+    public void commit(List<CommitMessage> commitMessages) {
+        checkState(!batchCommitted, "BatchTableCommit only support one-time committing.");
+        batchCommitted = true;
+        commit(BatchWriteBuilder.COMMIT_IDENTIFIER, commitMessages);
+    }
+
     @Override
     public void commit(long identifier, List<CommitMessage> commitMessages) {
+        commit(createManifestCommittable(identifier, commitMessages));
+    }
+
+    @Override
+    public int filterAndCommit(Map<Long, List<CommitMessage>> commitIdentifiersAndMessages) {
+        return filterAndCommitMultiple(
+                commitIdentifiersAndMessages.entrySet().stream()
+                        .map(e -> createManifestCommittable(e.getKey(), e.getValue()))
+                        .collect(Collectors.toList()));
+    }
+
+    private ManifestCommittable createManifestCommittable(
+            long identifier, List<CommitMessage> commitMessages) {
         ManifestCommittable committable = new ManifestCommittable(identifier);
         for (CommitMessage commitMessage : commitMessages) {
             committable.addFileCommittable(commitMessage);
         }
-        if (overwritePartition == null) {
-            commit.commit(committable, new HashMap<>());
-        } else {
-            commit.overwrite(overwritePartition, committable, Collections.emptyMap());
-        }
-        expire(identifier);
+        return committable;
     }
 
     public void commit(ManifestCommittable committable) {
@@ -141,6 +158,24 @@ public class TableCommitImpl implements InnerTableCommit {
         }
     }
 
+    public int filterAndCommitMultiple(List<ManifestCommittable> committables) {
+        Set<Long> retryIdentifiers =
+                commit.filterCommitted(
+                        committables.stream()
+                                .map(ManifestCommittable::identifier)
+                                .collect(Collectors.toSet()));
+        List<ManifestCommittable> retryCommittables =
+                committables.stream()
+                        .filter(c -> retryIdentifiers.contains(c.identifier()))
+                        // identifier must be in increasing order
+                        .sorted(Comparator.comparingLong(ManifestCommittable::identifier))
+                        .collect(Collectors.toList());
+        if (retryCommittables.size() > 0) {
+            commitMultiple(retryCommittables);
+        }
+        return retryCommittables.size();
+    }
+
     private void expire(long partitionExpireIdentifier) {
         // expire consumer first to avoid preventing snapshot expiration
         if (consumerExpireTime != null) {
@@ -163,13 +198,6 @@ public class TableCommitImpl implements InnerTableCommit {
         }
     }
 
-    @Override
-    public void commit(List<CommitMessage> commitMessages) {
-        checkState(!batchCommitted, "BatchTableCommit only support one-time committing.");
-        batchCommitted = true;
-        commit(BatchWriteBuilder.COMMIT_IDENTIFIER, commitMessages);
-    }
-
     @Override
     public void abort(List<CommitMessage> commitMessages) {
         commit.abort(commitMessages);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/Committer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/Committer.java
index 15b3d5098..482e1f098 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/Committer.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/Committer.java
@@ -32,10 +32,6 @@ import java.util.Map;
  */
 public interface Committer<CommitT, GlobalCommitT> extends AutoCloseable {
 
-    /** Find out which global committables need to be retried when recovering from the failure. */
-    List<GlobalCommitT> filterRecoveredCommittables(List<GlobalCommitT> globalCommittables)
-            throws IOException;
-
     /** Compute an aggregated committable from a list of committables. */
     GlobalCommitT combine(long checkpointId, long watermark, List<CommitT> committables)
             throws IOException;
@@ -43,5 +39,11 @@ public interface Committer<CommitT, GlobalCommitT> extends AutoCloseable {
     /** Commits the given {@link GlobalCommitT}. */
     void commit(List<GlobalCommitT> globalCommittables) throws IOException, InterruptedException;
 
+    /**
+     * Filter out all {@link GlobalCommitT} which have committed, and commit the remaining {@link
+     * GlobalCommitT}.
+     */
+    int filterAndCommit(List<GlobalCommitT> globalCommittables) throws IOException;
+
     Map<Long, List<CommitT>> groupByCheckpoint(Collection<CommitT> committables);
 }
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RestoreAndFailCommittableStateManager.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RestoreAndFailCommittableStateManager.java
index 42c7bcabd..b2401688e 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RestoreAndFailCommittableStateManager.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RestoreAndFailCommittableStateManager.java
@@ -78,9 +78,8 @@ public class RestoreAndFailCommittableStateManager<GlobalCommitT>
 
     private void recover(List<GlobalCommitT> committables, Committer<?, GlobalCommitT> committer)
             throws Exception {
-        committables = committer.filterRecoveredCommittables(committables);
-        if (!committables.isEmpty()) {
-            committer.commit(committables);
+        int numCommitted = committer.filterAndCommit(committables);
+        if (numCommitted > 0) {
             throw new RuntimeException(
                     "This exception is intentionally thrown "
                             + "after committing the restored checkpoints. "
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
index ed564e6e8..35e9229a1 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
@@ -29,8 +29,6 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
 
 /** {@link Committer} for dynamic store. */
 public class StoreCommitter implements Committer<Committable, ManifestCommittable> {
@@ -41,19 +39,6 @@ public class StoreCommitter implements Committer<Committable, ManifestCommittabl
         this.commit = (TableCommitImpl) commit;
     }
 
-    @Override
-    public List<ManifestCommittable> filterRecoveredCommittables(
-            List<ManifestCommittable> globalCommittables) {
-        Set<Long> identifiers =
-                commit.filterCommitted(
-                        globalCommittables.stream()
-                                .map(ManifestCommittable::identifier)
-                                .collect(Collectors.toSet()));
-        return globalCommittables.stream()
-                .filter(m -> identifiers.contains(m.identifier()))
-                .collect(Collectors.toList());
-    }
-
     @Override
     public ManifestCommittable combine(
             long checkpointId, long watermark, List<Committable> committables) {
@@ -80,6 +65,11 @@ public class StoreCommitter implements Committer<Committable, ManifestCommittabl
         commit.commitMultiple(committables);
     }
 
+    @Override
+    public int filterAndCommit(List<ManifestCommittable> globalCommittables) throws IOException {
+        return commit.filterAndCommitMultiple(globalCommittables);
+    }
+
     @Override
     public Map<Long, List<Committable>> groupByCheckpoint(Collection<Committable> committables) {
         Map<Long, List<Committable>> grouped = new HashMap<>();
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java
index f464edd16..1239d06ee 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java
@@ -31,10 +31,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 import java.util.stream.Collectors;
 
 /**
@@ -45,12 +43,12 @@ import java.util.stream.Collectors;
 public class StoreMultiCommitter
         implements Committer<MultiTableCommittable, WrappedManifestCommittable> {
 
-    private Catalog catalog;
+    private final Catalog catalog;
     private final String commitUser;
     // To make the commit behavior consistent with that of Committer,
     //    StoreMultiCommitter manages multiple committers which are
     //    referenced by table id.
-    private Map<Identifier, StoreCommitter> tableCommitters;
+    private final Map<Identifier, StoreCommitter> tableCommitters;
 
     public StoreMultiCommitter(String commitUser, Catalog.Loader catalogLoader) {
         this.catalog = catalogLoader.load();
@@ -58,34 +56,6 @@ public class StoreMultiCommitter
         this.tableCommitters = new HashMap<>();
     }
 
-    @Override
-    public List<WrappedManifestCommittable> filterRecoveredCommittables(
-            List<WrappedManifestCommittable> globalCommittables) {
-        // key by table id
-        Map<Identifier, List<ManifestCommittable>> committableMap =
-                groupByTable(globalCommittables);
-
-        Map<Long, WrappedManifestCommittable> result = new TreeMap<>();
-
-        for (Map.Entry<Identifier, List<ManifestCommittable>> entry : committableMap.entrySet()) {
-            Identifier tableId = entry.getKey();
-            List<ManifestCommittable> committableList = entry.getValue();
-            StoreCommitter committer = getStoreCommitter(tableId);
-            List<ManifestCommittable> filteredCommittables =
-                    committer.filterRecoveredCommittables(committableList);
-
-            for (ManifestCommittable filteredCommittable : filteredCommittables) {
-                long identifier = filteredCommittable.identifier();
-                WrappedManifestCommittable wrappedFilteredCommittable =
-                        result.computeIfAbsent(identifier, id -> new WrappedManifestCommittable());
-
-                wrappedFilteredCommittable.putManifestCommittable(tableId, filteredCommittable);
-            }
-        }
-
-        return new LinkedList<>(result.values());
-    }
-
     @Override
     public WrappedManifestCommittable combine(
             long checkpointId, long watermark, List<MultiTableCommittable> committables) {
@@ -128,6 +98,17 @@ public class StoreMultiCommitter
         }
     }
 
+    @Override
+    public int filterAndCommit(List<WrappedManifestCommittable> globalCommittables)
+            throws IOException {
+        int result = 0;
+        for (Map.Entry<Identifier, List<ManifestCommittable>> entry :
+                groupByTable(globalCommittables).entrySet()) {
+            result += getStoreCommitter(entry.getKey()).filterAndCommit(entry.getValue());
+        }
+        return result;
+    }
+
     private Map<Identifier, List<ManifestCommittable>> groupByTable(
             List<WrappedManifestCommittable> committables) {
         return committables.stream()