You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/06/21 08:17:35 UTC
[incubator-paimon] branch master updated: [core] Introduce filterAndCommit method for StreamTableCommit, which merges filterCommitted and commit (#1414)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3a91b6be2 [core] Introduce filterAndCommit method for StreamTableCommit, which merges filterCommitted and commit (#1414)
3a91b6be2 is described below
commit 3a91b6be2a755ba9b8a06ca2952c048676603aa2
Author: tsreaper <ts...@gmail.com>
AuthorDate: Wed Jun 21 16:17:31 2023 +0800
[core] Introduce filterAndCommit method for StreamTableCommit, which merges filterCommitted and commit (#1414)
---
docs/content/api/java-api.md | 12 +++--
.../paimon/table/sink/StreamTableCommit.java | 25 ++++++++++
.../paimon/table/sink/StreamWriteBuilder.java | 4 +-
.../apache/paimon/table/sink/TableCommitImpl.java | 54 ++++++++++++++++------
.../org/apache/paimon/flink/sink/Committer.java | 10 ++--
.../RestoreAndFailCommittableStateManager.java | 5 +-
.../apache/paimon/flink/sink/StoreCommitter.java | 20 ++------
.../paimon/flink/sink/StoreMultiCommitter.java | 45 ++++++------------
8 files changed, 103 insertions(+), 72 deletions(-)
diff --git a/docs/content/api/java-api.md b/docs/content/api/java-api.md
index 48c324f5e..9ddfcc0f4 100644
--- a/docs/content/api/java-api.md
+++ b/docs/content/api/java-api.md
@@ -560,7 +560,7 @@ Key points to achieve exactly-once consistency:
- Different applications need to use different commitUsers.
- The commitIdentifier of `StreamTableWrite` and `StreamTableCommit` needs to be consistent, and the
id needs to be incremented for the next committing.
-- When a failure occurs, if you still have uncommitted `CommitMessage`s, please use `StreamTableCommit#filterCommitted`
+- When a failure occurs, if you still have uncommitted `CommitMessage`s, please use `StreamTableCommit#filterAndCommit`
to exclude the committed messages by commitIdentifier.
```java
@@ -594,8 +594,14 @@ public class StreamWriteTable {
StreamTableCommit commit = writeBuilder.newCommit();
commit.commit(commitIdentifier, messages);
- // 4. When failover, you can use 'filterCommitted' to filter committed commits.
- commit.filterCommitted(committedIdentifiers);
+ // 4. When failure occurs and you're not sure if the commit process is successful,
+ // you can use `filterAndCommit` to retry the commit process.
+ // Succeeded commits will be automatically skipped.
+ /*
+ Map<Long, List<CommitMessage>> commitIdentifiersAndMessages = new HashMap<>();
+ commitIdentifiersAndMessages.put(commitIdentifier, messages);
+ commit.filterAndCommit(commitIdentifiersAndMessages);
+ */
}
}
```
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamTableCommit.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamTableCommit.java
index 6a4dc4a25..d2c91dc40 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamTableCommit.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamTableCommit.java
@@ -21,6 +21,7 @@ package org.apache.paimon.table.sink;
import org.apache.paimon.annotation.Public;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -35,7 +36,12 @@ public interface StreamTableCommit extends TableCommit {
/**
* Filter committed commits. Return uncommitted identifiers. This method is used for failover
* cases.
+ *
+ * @deprecated Use {@link StreamTableCommit#filterAndCommit} to filter and commit all {@link
+ * CommitMessage} in question with one method call, instead of calling this method first and
+ * then call {@link StreamTableCommit#commit}.
*/
+ @Deprecated
Set<Long> filterCommitted(Set<Long> commitIdentifiers);
/**
@@ -55,10 +61,29 @@ public interface StreamTableCommit extends TableCommit {
* The check frequency is controlled by 'partition.expiration-check-interval'. Partition
* expiration will create an 'OVERWRITE' snapshot.
*
+ * <p>Compared to {@link StreamTableCommit#filterAndCommit}, this method does not check if
+ * {@code commitIdentifier} has been committed, so this method might be faster. Please only use
+ * this method if you can make sure that the {@code commitIdentifier} hasn't been committed
+ * before.
+ *
* @param commitIdentifier Committed transaction ID, can start from 0. If there are multiple
* commits, please increment this ID.
* @param commitMessages commit messages from table write
* @see StreamTableWrite#prepareCommit
*/
void commit(long commitIdentifier, List<CommitMessage> commitMessages);
+
+ /**
+ * Filter out all {@code List<CommitMessage>} which have been committed and commit the remaining
+ * ones.
+ *
+ * <p>Compared to {@link StreamTableCommit#commit}, this method will first check if a {@code
+ * commitIdentifier} has been committed, so this method might be slower. A common usage of this
+ * method is to retry the commit process after a failure.
+ *
+ * @param commitIdentifiersAndMessages a map containing all {@link CommitMessage}s in question.
+ * The key is the {@code commitIdentifier}.
+ * @return number of {@code List<CommitMessage>} committed.
+ */
+ int filterAndCommit(Map<Long, List<CommitMessage>> commitIdentifiersAndMessages);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamWriteBuilder.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamWriteBuilder.java
index c5465de2d..7f272e075 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamWriteBuilder.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/StreamWriteBuilder.java
@@ -32,8 +32,8 @@ import org.apache.paimon.data.InternalRow;
* <li>Different applications need to use different commitUsers.
* <li>The commitIdentifier of {@link StreamTableWrite} and {@link StreamTableCommit} needs to be
* consistent, and the id needs to be incremented for the next committing.
- * <li>When a failure occurs, if you still have uncommitted {@link CommitMessage}s, please use
- * {@link StreamTableCommit#filterCommitted} to exclude the committed messages by
+ * <li>When a failure occurs, if you still have uncommitted {@link CommitMessage}s, please retry
+ * with {@link StreamTableCommit#filterAndCommit} to exclude the committed messages by
* commitIdentifier.
* </ul>
*
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
index 96913e1fc..a25823a0d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
@@ -30,10 +30,12 @@ import javax.annotation.Nullable;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import static org.apache.paimon.utils.Preconditions.checkState;
@@ -96,18 +98,33 @@ public class TableCommitImpl implements InnerTableCommit {
return commit.filterCommitted(commitIdentifiers);
}
+ @Override
+ public void commit(List<CommitMessage> commitMessages) {
+ checkState(!batchCommitted, "BatchTableCommit only support one-time committing.");
+ batchCommitted = true;
+ commit(BatchWriteBuilder.COMMIT_IDENTIFIER, commitMessages);
+ }
+
@Override
public void commit(long identifier, List<CommitMessage> commitMessages) {
+ commit(createManifestCommittable(identifier, commitMessages));
+ }
+
+ @Override
+ public int filterAndCommit(Map<Long, List<CommitMessage>> commitIdentifiersAndMessages) {
+ return filterAndCommitMultiple(
+ commitIdentifiersAndMessages.entrySet().stream()
+ .map(e -> createManifestCommittable(e.getKey(), e.getValue()))
+ .collect(Collectors.toList()));
+ }
+
+ private ManifestCommittable createManifestCommittable(
+ long identifier, List<CommitMessage> commitMessages) {
ManifestCommittable committable = new ManifestCommittable(identifier);
for (CommitMessage commitMessage : commitMessages) {
committable.addFileCommittable(commitMessage);
}
- if (overwritePartition == null) {
- commit.commit(committable, new HashMap<>());
- } else {
- commit.overwrite(overwritePartition, committable, Collections.emptyMap());
- }
- expire(identifier);
+ return committable;
}
public void commit(ManifestCommittable committable) {
@@ -141,6 +158,24 @@ public class TableCommitImpl implements InnerTableCommit {
}
}
+ public int filterAndCommitMultiple(List<ManifestCommittable> committables) {
+ Set<Long> retryIdentifiers =
+ commit.filterCommitted(
+ committables.stream()
+ .map(ManifestCommittable::identifier)
+ .collect(Collectors.toSet()));
+ List<ManifestCommittable> retryCommittables =
+ committables.stream()
+ .filter(c -> retryIdentifiers.contains(c.identifier()))
+ // identifier must be in increasing order
+ .sorted(Comparator.comparingLong(ManifestCommittable::identifier))
+ .collect(Collectors.toList());
+ if (retryCommittables.size() > 0) {
+ commitMultiple(retryCommittables);
+ }
+ return retryCommittables.size();
+ }
+
private void expire(long partitionExpireIdentifier) {
// expire consumer first to avoid preventing snapshot expiration
if (consumerExpireTime != null) {
@@ -163,13 +198,6 @@ public class TableCommitImpl implements InnerTableCommit {
}
}
- @Override
- public void commit(List<CommitMessage> commitMessages) {
- checkState(!batchCommitted, "BatchTableCommit only support one-time committing.");
- batchCommitted = true;
- commit(BatchWriteBuilder.COMMIT_IDENTIFIER, commitMessages);
- }
-
@Override
public void abort(List<CommitMessage> commitMessages) {
commit.abort(commitMessages);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/Committer.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/Committer.java
index 15b3d5098..482e1f098 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/Committer.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/Committer.java
@@ -32,10 +32,6 @@ import java.util.Map;
*/
public interface Committer<CommitT, GlobalCommitT> extends AutoCloseable {
- /** Find out which global committables need to be retried when recovering from the failure. */
- List<GlobalCommitT> filterRecoveredCommittables(List<GlobalCommitT> globalCommittables)
- throws IOException;
-
/** Compute an aggregated committable from a list of committables. */
GlobalCommitT combine(long checkpointId, long watermark, List<CommitT> committables)
throws IOException;
@@ -43,5 +39,11 @@ public interface Committer<CommitT, GlobalCommitT> extends AutoCloseable {
/** Commits the given {@link GlobalCommitT}. */
void commit(List<GlobalCommitT> globalCommittables) throws IOException, InterruptedException;
+ /**
+ * Filter out all {@link GlobalCommitT} which have committed, and commit the remaining {@link
+ * GlobalCommitT}.
+ */
+ int filterAndCommit(List<GlobalCommitT> globalCommittables) throws IOException;
+
Map<Long, List<CommitT>> groupByCheckpoint(Collection<CommitT> committables);
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RestoreAndFailCommittableStateManager.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RestoreAndFailCommittableStateManager.java
index 42c7bcabd..b2401688e 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RestoreAndFailCommittableStateManager.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/RestoreAndFailCommittableStateManager.java
@@ -78,9 +78,8 @@ public class RestoreAndFailCommittableStateManager<GlobalCommitT>
private void recover(List<GlobalCommitT> committables, Committer<?, GlobalCommitT> committer)
throws Exception {
- committables = committer.filterRecoveredCommittables(committables);
- if (!committables.isEmpty()) {
- committer.commit(committables);
+ int numCommitted = committer.filterAndCommit(committables);
+ if (numCommitted > 0) {
throw new RuntimeException(
"This exception is intentionally thrown "
+ "after committing the restored checkpoints. "
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
index ed564e6e8..35e9229a1 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
@@ -29,8 +29,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
/** {@link Committer} for dynamic store. */
public class StoreCommitter implements Committer<Committable, ManifestCommittable> {
@@ -41,19 +39,6 @@ public class StoreCommitter implements Committer<Committable, ManifestCommittabl
this.commit = (TableCommitImpl) commit;
}
- @Override
- public List<ManifestCommittable> filterRecoveredCommittables(
- List<ManifestCommittable> globalCommittables) {
- Set<Long> identifiers =
- commit.filterCommitted(
- globalCommittables.stream()
- .map(ManifestCommittable::identifier)
- .collect(Collectors.toSet()));
- return globalCommittables.stream()
- .filter(m -> identifiers.contains(m.identifier()))
- .collect(Collectors.toList());
- }
-
@Override
public ManifestCommittable combine(
long checkpointId, long watermark, List<Committable> committables) {
@@ -80,6 +65,11 @@ public class StoreCommitter implements Committer<Committable, ManifestCommittabl
commit.commitMultiple(committables);
}
+ @Override
+ public int filterAndCommit(List<ManifestCommittable> globalCommittables) throws IOException {
+ return commit.filterAndCommitMultiple(globalCommittables);
+ }
+
@Override
public Map<Long, List<Committable>> groupByCheckpoint(Collection<Committable> committables) {
Map<Long, List<Committable>> grouped = new HashMap<>();
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java
index f464edd16..1239d06ee 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreMultiCommitter.java
@@ -31,10 +31,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import java.util.stream.Collectors;
/**
@@ -45,12 +43,12 @@ import java.util.stream.Collectors;
public class StoreMultiCommitter
implements Committer<MultiTableCommittable, WrappedManifestCommittable> {
- private Catalog catalog;
+ private final Catalog catalog;
private final String commitUser;
// To make the commit behavior consistent with that of Committer,
// StoreMultiCommitter manages multiple committers which are
// referenced by table id.
- private Map<Identifier, StoreCommitter> tableCommitters;
+ private final Map<Identifier, StoreCommitter> tableCommitters;
public StoreMultiCommitter(String commitUser, Catalog.Loader catalogLoader) {
this.catalog = catalogLoader.load();
@@ -58,34 +56,6 @@ public class StoreMultiCommitter
this.tableCommitters = new HashMap<>();
}
- @Override
- public List<WrappedManifestCommittable> filterRecoveredCommittables(
- List<WrappedManifestCommittable> globalCommittables) {
- // key by table id
- Map<Identifier, List<ManifestCommittable>> committableMap =
- groupByTable(globalCommittables);
-
- Map<Long, WrappedManifestCommittable> result = new TreeMap<>();
-
- for (Map.Entry<Identifier, List<ManifestCommittable>> entry : committableMap.entrySet()) {
- Identifier tableId = entry.getKey();
- List<ManifestCommittable> committableList = entry.getValue();
- StoreCommitter committer = getStoreCommitter(tableId);
- List<ManifestCommittable> filteredCommittables =
- committer.filterRecoveredCommittables(committableList);
-
- for (ManifestCommittable filteredCommittable : filteredCommittables) {
- long identifier = filteredCommittable.identifier();
- WrappedManifestCommittable wrappedFilteredCommittable =
- result.computeIfAbsent(identifier, id -> new WrappedManifestCommittable());
-
- wrappedFilteredCommittable.putManifestCommittable(tableId, filteredCommittable);
- }
- }
-
- return new LinkedList<>(result.values());
- }
-
@Override
public WrappedManifestCommittable combine(
long checkpointId, long watermark, List<MultiTableCommittable> committables) {
@@ -128,6 +98,17 @@ public class StoreMultiCommitter
}
}
+ @Override
+ public int filterAndCommit(List<WrappedManifestCommittable> globalCommittables)
+ throws IOException {
+ int result = 0;
+ for (Map.Entry<Identifier, List<ManifestCommittable>> entry :
+ groupByTable(globalCommittables).entrySet()) {
+ result += getStoreCommitter(entry.getKey()).filterAndCommit(entry.getValue());
+ }
+ return result;
+ }
+
private Map<Identifier, List<ManifestCommittable>> groupByTable(
List<WrappedManifestCommittable> committables) {
return committables.stream()