You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ec...@apache.org on 2019/05/28 07:33:18 UTC
[beam] 03/06: Add comment about checkpoint mark
This is an automated email from the ASF dual-hosted git repository.
echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 6ba3d1cb0d09ea0aa7f6a5f308fd59ccceae9d47
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Wed May 15 14:24:11 2019 +0200
Add comment about checkpoint mark
---
.../translation/streaming/DatasetSourceStreaming.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java
index 234eecf..1cf52ba 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java
@@ -134,6 +134,7 @@ class DatasetSourceStreaming implements DataSourceV2, MicroBatchReadSupport {
// offsets are ignored see javadoc
for (DatasetPartitionReader partitionReader : partitionReaders) {
try {
+ // TODO: is checkpointMark stored in reliable storage ?
partitionReader.reader.getCheckpointMark().finalizeCheckpoint();
} catch (IOException e) {
throw new RuntimeException(