You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2023/05/17 03:16:35 UTC
[beam] branch master updated: Update Bigtable Change Stream Connector to include latest features (#26454)
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d5c89523938 Update Bigtable Change Stream Connector to include latest features (#26454)
d5c89523938 is described below
commit d5c895239385359edac3b4b96304f1b5507f2774
Author: Tony Tang <nf...@gmail.com>
AuthorDate: Tue May 16 23:16:26 2023 -0400
Update Bigtable Change Stream Connector to include latest features (#26454)
* Include latest changes from local development
* Reconciler update to reuse continuation tokens to reduce the
likelihood of missing mutations
* StreamPartition and NewPartition metadata are performed as 2 phase
deletes to reduce race conditions that loses tokens
* Pipelines can be drained and restarted using the same
ChangeStreamName
* Added OrphanCleaner to clean up lingering NewPartition metadata
* General refactoring to improve readability
* Fix javadoc format issue
---
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 84 ++-
.../changestreams/ByteStringRangeHelper.java | 175 ++++---
.../changestreams/ChangeStreamMetrics.java | 111 ++--
.../bigtable/changestreams/TimestampConverter.java | 4 +
.../bigtable/changestreams/UniqueIdGenerator.java | 19 +-
.../changestreams/action/ActionFactory.java | 35 +-
.../changestreams/action/ChangeStreamAction.java | 6 +-
.../action/DetectNewPartitionsAction.java | 406 ++++++---------
.../action/GenerateInitialPartitionsAction.java | 30 +-
.../action/ProcessNewPartitionsAction.java | 116 +++++
.../action/ReadChangeStreamPartitionAction.java | 101 +++-
.../action/ResumeFromPreviousPipelineAction.java | 90 ++++
.../dao/BigtableChangeStreamAccessor.java | 9 -
.../changestreams/dao/ChangeStreamDao.java | 1 -
.../gcp/bigtable/changestreams/dao/DaoFactory.java | 7 +-
.../changestreams/dao/MetadataTableAdminDao.java | 36 +-
.../changestreams/dao/MetadataTableDao.java | 535 ++++++++++++++++---
.../dofn/DetectNewPartitionsDoFn.java | 32 +-
.../changestreams/dofn/InitializeDoFn.java | 74 ++-
.../encoder/MetadataTableEncoder.java | 66 ++-
.../model/DetectNewPartitionsState.java | 78 +++
.../changestreams/model/InitialPipelineState.java | 70 +++
.../bigtable/changestreams/model/NewPartition.java | 146 ++++++
.../changestreams/model/PartitionRecord.java | 84 +--
.../model/StreamPartitionWithWatermark.java | 76 +++
.../reconciler/OrphanedMetadataCleaner.java | 87 ++++
.../reconciler/PartitionReconciler.java | 183 +++++--
.../restriction/DetectNewPartitionsTracker.java | 53 ++
.../ReadChangeStreamPartitionProgressTracker.java | 11 +-
.../changestreams/restriction/StreamProgress.java | 6 +-
.../changestreams/ByteStringRangeHelperTest.java | 262 +++++++---
.../changestreams/TimestampConverterTest.java | 6 +
.../action/ChangeStreamActionTest.java | 8 +-
.../action/DetectNewPartitionsActionTest.java | 570 ++++++++++++++++-----
.../GenerateInitialPartitionsActionTest.java | 22 +-
.../ReadChangeStreamPartitionActionTest.java | 184 +++++--
.../ResumeFromPreviousPipelineActionTest.java | 288 +++++++++++
.../changestreams/dao/MetadataTableDaoTest.java | 479 ++++++++++++++---
.../changestreams/dofn/InitializeDoFnTest.java | 208 +++++++-
.../dofn/ReadChangeStreamPartitionDoFnTest.java | 11 +-
.../reconciler/OrphanedMetadataCleanerTest.java | 82 +++
.../reconciler/PartitionReconcilerTest.java | 259 +++++++---
.../DetectNewPartitionsTrackerTest.java | 45 ++
...adChangeStreamPartitionProgressTrackerTest.java | 5 +-
44 files changed, 4092 insertions(+), 1068 deletions(-)
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 88e94f03140..fc8b5e78e8a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -1782,6 +1782,27 @@ public class BigtableIO {
cause);
}
}
+ /**
+ * Overwrite options to determine what to do if change stream name is being reused and there
+ * exists metadata of the same change stream name.
+ */
+ public enum ExistingPipelineOptions {
+ // Don't start if there exists metadata of the same change stream name.
+ FAIL_IF_EXISTS,
+ // Pick up from where the previous pipeline left off. This will perform resumption at best
+ // effort guaranteeing, at-least-once delivery. So it's likely that duplicate data, seen before
+ // the pipeline was stopped, will be outputted. If previous pipeline doesn't exist, start a new
+ // pipeline.
+ RESUME_OR_NEW,
+ // Same as RESUME_OR_NEW except if previous pipeline doesn't exist, don't start.
+ RESUME_OR_FAIL,
+ // Start a new pipeline. Overriding existing pipeline with the same name.
+ NEW,
+ // This skips cleaning up previous pipeline metadata and starts a new pipeline. This should
+ // only be used to skip cleanup in tests
+ @VisibleForTesting
+ SKIP_CLEANUP,
+ }
@AutoValue
public abstract static class ReadChangeStream
@@ -1809,11 +1830,14 @@ public class BigtableIO {
abstract @Nullable String getChangeStreamName();
+ abstract @Nullable ExistingPipelineOptions getExistingPipelineOptions();
+
abstract BigtableConfig getMetadataTableBigtableConfig();
abstract @Nullable String getMetadataTableId();
abstract ReadChangeStream.Builder toBuilder();
+
/**
* Returns a new {@link BigtableIO.ReadChangeStream} that will stream from the Cloud Bigtable
* project indicated by given parameter, requires {@link #withInstanceId} to be called to
@@ -1904,6 +1928,17 @@ public class BigtableIO {
return toBuilder().setChangeStreamName(changeStreamName).build();
}
+ /**
+ * Returns a new {@link BigtableIO.ReadChangeStream} that decides what to do if an existing
+ * pipeline exists with the same change stream name.
+ *
+ * <p>Does not modify this object.
+ */
+ public ReadChangeStream withExistingPipelineOptions(
+ ExistingPipelineOptions existingPipelineOptions) {
+ return toBuilder().setExistingPipelineOptions(existingPipelineOptions).build();
+ }
+
/**
* Returns a new {@link BigtableIO.ReadChangeStream} that will use the Cloud Bigtable project
* indicated by given parameter to manage the metadata of the stream.
@@ -1966,30 +2001,38 @@ public class BigtableIO {
@Override
public PCollection<KV<ByteString, ChangeStreamMutation>> expand(PBegin input) {
- checkArgument(getBigtableConfig() != null);
- checkArgument(getBigtableConfig().getProjectId() != null);
- checkArgument(getBigtableConfig().getInstanceId() != null);
- checkArgument(getBigtableConfig().getAppProfileId() != null);
- checkArgument(getTableId() != null);
+ checkArgument(
+ getBigtableConfig() != null,
+ "BigtableIO ReadChangeStream is missing required configurations fields.");
+ checkArgument(
+ getBigtableConfig().getProjectId() != null, "Missing required projectId field.");
+ checkArgument(
+ getBigtableConfig().getInstanceId() != null, "Missing required instanceId field.");
+ checkArgument(getTableId() != null, "Missing required tableId field.");
+
+ BigtableConfig bigtableConfig = getBigtableConfig();
+ if (getBigtableConfig().getAppProfileId() == null
+ || getBigtableConfig().getAppProfileId().get().isEmpty()) {
+ bigtableConfig = bigtableConfig.withAppProfileId(StaticValueProvider.of("default"));
+ }
BigtableConfig metadataTableConfig = getMetadataTableBigtableConfig();
+ String metadataTableId = getMetadataTableId();
if (metadataTableConfig.getProjectId() == null
|| metadataTableConfig.getProjectId().get().isEmpty()) {
- metadataTableConfig = metadataTableConfig.withProjectId(getBigtableConfig().getProjectId());
+ metadataTableConfig = metadataTableConfig.withProjectId(bigtableConfig.getProjectId());
}
if (metadataTableConfig.getInstanceId() == null
|| metadataTableConfig.getInstanceId().get().isEmpty()) {
- metadataTableConfig =
- metadataTableConfig.withInstanceId(getBigtableConfig().getInstanceId());
+ metadataTableConfig = metadataTableConfig.withInstanceId(bigtableConfig.getInstanceId());
}
- String metadataTableId = getMetadataTableId();
if (metadataTableId == null || metadataTableId.isEmpty()) {
metadataTableId = MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME;
}
if (metadataTableConfig.getAppProfileId() == null
|| metadataTableConfig.getAppProfileId().get().isEmpty()) {
metadataTableConfig =
- metadataTableConfig.withAppProfileId(getBigtableConfig().getAppProfileId());
+ metadataTableConfig.withAppProfileId(bigtableConfig.getAppProfileId());
}
Instant startTime = getStartTime();
@@ -2004,18 +2047,22 @@ public class BigtableIO {
if (changeStreamName == null || changeStreamName.isEmpty()) {
changeStreamName = UniqueIdGenerator.generateRowKeyPrefix();
}
+ ExistingPipelineOptions existingPipelineOptions = getExistingPipelineOptions();
+ if (existingPipelineOptions == null) {
+ existingPipelineOptions = ExistingPipelineOptions.FAIL_IF_EXISTS;
+ }
ActionFactory actionFactory = new ActionFactory();
DaoFactory daoFactory =
new DaoFactory(
- getBigtableConfig(),
- metadataTableConfig,
- getTableId(),
- metadataTableId,
- changeStreamName);
+ bigtableConfig, metadataTableConfig, getTableId(), metadataTableId, changeStreamName);
ChangeStreamMetrics metrics = new ChangeStreamMetrics();
InitializeDoFn initializeDoFn =
- new InitializeDoFn(daoFactory, metadataTableConfig.getAppProfileId().get(), startTime);
+ new InitializeDoFn(
+ daoFactory,
+ metadataTableConfig.getAppProfileId().get(),
+ startTime,
+ existingPipelineOptions);
DetectNewPartitionsDoFn detectNewPartitionsDoFn =
new DetectNewPartitionsDoFn(getEndTime(), actionFactory, daoFactory, metrics);
ReadChangeStreamPartitionDoFn readChangeStreamPartitionDoFn =
@@ -2049,7 +2096,7 @@ public class BigtableIO {
abstract ReadChangeStream.Builder setMetadataTableBigtableConfig(
BigtableConfig bigtableConfig);
- abstract ReadChangeStream.Builder setMetadataTableId(String tableId);
+ abstract ReadChangeStream.Builder setMetadataTableId(String metadataTableId);
abstract ReadChangeStream.Builder setStartTime(Instant startTime);
@@ -2059,6 +2106,9 @@ public class BigtableIO {
abstract ReadChangeStream.Builder setChangeStreamName(String changeStreamName);
+ abstract ReadChangeStream.Builder setExistingPipelineOptions(
+ ExistingPipelineOptions existingPipelineOptions);
+
abstract ReadChangeStream build();
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java
index 54285a5b332..900ddae570e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java
@@ -45,6 +45,13 @@ public class ByteStringRangeHelper {
return ByteString.unsignedLexicographicalComparator().compare(e1, e2);
}
+ private static int compareStartEndKey(ByteString start, ByteString end) {
+ if (end.isEmpty()) {
+ return -1;
+ }
+ return ByteString.unsignedLexicographicalComparator().compare(start, end);
+ }
+
@VisibleForTesting
static class PartitionComparator implements Comparator<ByteStringRange> {
@Override
@@ -65,36 +72,37 @@ public class ByteStringRangeHelper {
}
/**
- * Returns true if parentPartitions is a superset of childPartition.
+ * Returns true if parentPartitions form a proper superset of childPartition.
*
- * <p>If ordered parentPartitions row ranges form a contiguous range, and start key is before or
- * at childPartition's start key, and end key is at or after childPartition's end key, then
- * parentPartitions is a superset of childPartition.
+ * <p>If ordered parentPartitions row ranges form a contiguous range without any overlaps, and
+ * start key equals to childPartition's start key, and end key equals to childPartition's end key,
+ * then parentPartitions form a proper superset of childPartition.
*
- * <p>Overlaps from parents are valid because arbitrary partitions can merge and they may overlap.
- * They will form a valid new partition. However, if there are any missing parent partitions, then
- * merge cannot happen with missing row ranges.
+ * <p>Returns false if there are overlaps or missing partitions in parentPartitions.
*
- * @param parentPartitions list of partitions to determine if it forms a large contiguous range
- * @param childPartition the smaller partition
- * @return true if parentPartitions is a superset of childPartition, otherwise false.
+ * @param parentPartitions list of partitions to determine if it forms an exact contiguous range
+ * @param childPartition the partition to match
+ * @return true if parentPartitions covers the same key space as childPartition, otherwise false.
*/
- public static boolean isSuperset(
+ public static boolean coverSameKeySpace(
List<ByteStringRange> parentPartitions, ByteStringRange childPartition) {
- // sort parentPartitions by starting key
- // iterate through, check open end key and close start key of each iteration to ensure no gaps.
- // first start key and last end key must be equal to or wider than child partition start and end
- // key.
+ // Sort parentPartitions by starting key. Verify there's no gap or overlaps in between parent
+ // partitions.
+ // The first start key and last end key must be equal to child partition.
if (parentPartitions.isEmpty()) {
return false;
}
parentPartitions.sort(new PartitionComparator());
- ByteString parentStartKey = parentPartitions.get(0).getStart();
- ByteString parentEndKey = parentPartitions.get(parentPartitions.size() - 1).getEnd();
-
- return !childStartsBeforeParent(parentStartKey, childPartition.getStart())
- && !childEndsAfterParent(parentEndKey, childPartition.getEnd())
- && !gapsInParentPartitions(parentPartitions);
+ if (!parentPartitions.get(0).getStart().equals(childPartition.getStart())) {
+ return false;
+ }
+ if (!parentPartitions
+ .get(parentPartitions.size() - 1)
+ .getEnd()
+ .equals(childPartition.getEnd())) {
+ return false;
+ }
+ return isContinuous(parentPartitions);
}
/**
@@ -110,20 +118,18 @@ public class ByteStringRangeHelper {
}
/**
- * Figure out if partitions cover the entire keyspace. If it doesn't, return a list of missing and
- * overlapping partitions.
+ * Return missing partitions within partitions that are within start and end. Gaps in partitions
+ * that are outside of start and end are ignored.
*
- * <p>partitions covers the entire key space if, when ordered, the end key is the same as the
- * start key of the next row range in the list, and the first start key is "" and the last end key
- * is "". There should be no overlap.
- *
- * @param partitions to determine if they cover entire keyspace
- * @return list of missing and overlapping partitions
+ * @param partitions find missing partitions within
+ * @param start start of the search range
+ * @param end end of the search range
+ * @return missing partitions within the range
*/
- public static List<ByteStringRange> getMissingAndOverlappingPartitionsFromKeySpace(
- List<ByteStringRange> partitions) {
+ public static List<ByteStringRange> getMissingPartitionsFrom(
+ List<ByteStringRange> partitions, ByteString start, ByteString end) {
if (partitions.isEmpty()) {
- return Collections.singletonList(ByteStringRange.create("", ""));
+ return Collections.singletonList(ByteStringRange.create(start, end));
}
List<ByteStringRange> missingPartitions = new ArrayList<>();
@@ -132,22 +138,81 @@ public class ByteStringRangeHelper {
// iterate through ensuring end key is lexicographically after next start key.
partitions.sort(new PartitionComparator());
- ByteString prevEnd = ByteString.EMPTY;
- for (ByteStringRange partition : partitions) {
- if (!partition.getStart().equals(prevEnd)) {
- ByteStringRange missingPartition = ByteStringRange.create(prevEnd, partition.getStart());
- missingPartitions.add(missingPartition);
+ // Check if the start is missing.
+ if (compareStartKey(start, partitions.get(0).getStart()) < 0) {
+ missingPartitions.add(ByteStringRange.create(start, partitions.get(0).getStart()));
+ }
+ ByteString currEnd = partitions.get(0).getEnd();
+ if (compareStartKey(start, partitions.get(0).getEnd()) > 0) {
+ currEnd = start;
+ }
+ for (int i = 1; i < partitions.size(); i++) {
+ if (compareStartEndKey(partitions.get(i).getStart(), currEnd) > 0) {
+ missingPartitions.add(ByteStringRange.create(currEnd, partitions.get(i).getStart()));
+ }
+ // It's possible that we have subset partitions that has a later start key but earlier end
+ // key. So we only advance end key if it's further ahead. For example [A, D) and [B, C).
+ // PrevEnd = D and processing [B, C) would push back end key to C instead of D.
+ if (compareEndKey(currEnd, partitions.get(i).getEnd()) < 0) {
+ currEnd = partitions.get(i).getEnd();
+ }
+ // Current end maybe before the start of the evaluation range.
+ if (compareStartKey(start, currEnd) > 0) {
+ currEnd = start;
+ }
+ // If it's past the end, we stop
+ if (compareEndKey(currEnd, end) > 0) {
+ break;
}
- prevEnd = partition.getEnd();
}
- // Check that the last partition ends with "", otherwise it's missing.
- if (!prevEnd.equals(ByteString.EMPTY)) {
- ByteStringRange missingPartition = ByteStringRange.create(prevEnd, ByteString.EMPTY);
- missingPartitions.add(missingPartition);
+ // Check that the last partition ends with end, otherwise it's missing.
+ if (compareEndKey(currEnd, end) < 0) {
+ missingPartitions.add(ByteStringRange.create(currEnd, end));
}
return missingPartitions;
}
+ /**
+ * Return missing partitions from the entire keyspace.
+ *
+ * @param partitions find missing partitions within
+ * @return a list of missing partitions.
+ */
+ public static List<ByteStringRange> getMissingPartitionsFromEntireKeySpace(
+ List<ByteStringRange> partitions) {
+ return getMissingPartitionsFrom(partitions, ByteString.EMPTY, ByteString.EMPTY);
+ }
+
+ /**
+ * Return a list of overlapping partitions. The partitions outputted are valid partitions i.e.
+ * start < end. This represents 2 ordered partitions overlapping because the 2nd partition starts
+ * before the 1st ended. If end == start, it's not considered overlap.
+ *
+ * @param partitions find overlaps within
+ * @return a list of overlapping partitions.
+ */
+ public static List<ByteStringRange> getOverlappingPartitions(List<ByteStringRange> partitions) {
+ if (partitions.isEmpty() || partitions.size() == 1) {
+ return Collections.emptyList();
+ }
+
+ List<ByteStringRange> overlappingPartitions = new ArrayList<>();
+
+ // sort partitions by start key
+ // iterate through ensuring end key is lexicographically after next start key.
+ partitions.sort(new PartitionComparator());
+
+ // Determine where we start
+ ByteString prevEnd = partitions.get(0).getEnd();
+ for (int i = 1; i < partitions.size(); i++) {
+ if (compareStartEndKey(partitions.get(i).getStart(), prevEnd) < 0) {
+ overlappingPartitions.add(ByteStringRange.create(partitions.get(i).getStart(), prevEnd));
+ }
+ prevEnd = partitions.get(i).getEnd();
+ }
+ return overlappingPartitions;
+ }
+
/**
* Returns formatted string of a partition for debugging.
*
@@ -162,32 +227,18 @@ public class ByteStringRangeHelper {
+ "')";
}
- private static boolean childStartsBeforeParent(
- ByteString parentStartKey, ByteString childStartKey) {
- // Check if the start key of the child partition comes before the start key of the entire
- // parentPartitions
- return compareStartKey(parentStartKey, childStartKey) > 0;
- }
-
- private static boolean childEndsAfterParent(ByteString parentEndKey, ByteString childEndKey) {
- return compareEndKey(parentEndKey, childEndKey) < 0;
- }
-
// This assumes parentPartitions is sorted. If parentPartitions has not already been sorted
// it will be incorrect
- private static boolean gapsInParentPartitions(List<ByteStringRange> sortedParentPartitions) {
+ private static boolean isContinuous(List<ByteStringRange> sortedParentPartitions) {
for (int i = 1; i < sortedParentPartitions.size(); i++) {
- // Iterating through a sorted list, the start key should be the same or before the end of the
- // previous. Handle "" end key as a special case.
+ // Iterating through a sorted list, the start key should be the same as the end of the
+ // previous.
ByteString prevEndKey = sortedParentPartitions.get(i - 1).getEnd();
- if (ByteString.unsignedLexicographicalComparator()
- .compare(sortedParentPartitions.get(i).getStart(), prevEndKey)
- > 0
- && !prevEndKey.isEmpty()) {
- return true;
+ if (!sortedParentPartitions.get(i).getStart().equals(prevEndKey)) {
+ return false;
}
}
- return false;
+ return true;
}
/**
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ChangeStreamMetrics.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ChangeStreamMetrics.java
index b3548961f43..f732a52fd37 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ChangeStreamMetrics.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ChangeStreamMetrics.java
@@ -34,27 +34,33 @@ public class ChangeStreamMetrics implements Serializable {
* Counter for the total number of partitions identified during the execution of the Connector.
*/
public static final Counter LIST_PARTITIONS_COUNT =
- Metrics.counter(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "list_partitions_count");
+ Metrics.counter(ChangeStreamMetrics.class, "list_partitions_count");
/**
* Counter for the total number of partition splits / moves identified during the execution of the
* Connector.
*/
public static final Counter PARTITION_SPLIT_COUNT =
- Metrics.counter(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "partition_record_split_count");
+ Metrics.counter(ChangeStreamMetrics.class, "partition_record_split_count");
/**
* Counter for the total number of partition merges identified during the execution of the
* Connector.
*/
public static final Counter PARTITION_MERGE_COUNT =
- Metrics.counter(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "partition_record_merge_count");
+ Metrics.counter(ChangeStreamMetrics.class, "partition_record_merge_count");
+
+ /** Counter for the total number of partitions reconciled with continuation tokens. */
+ public static final Counter PARTITION_RECONCILED_WITH_TOKEN_COUNT =
+ Metrics.counter(ChangeStreamMetrics.class, "partition_reconciled_with_token_count");
+
+ /** Counter for the total number of partitions reconciled without continuation tokens. */
+ public static final Counter PARTITION_RECONCILED_WITHOUT_TOKEN_COUNT =
+ Metrics.counter(ChangeStreamMetrics.class, "partition_reconciled_without_token_count");
+
+ /** Counter for the total number of orphaned new partitions cleaned up. */
+ public static final Counter ORPHANED_NEW_PARTITION_CLEANED_COUNT =
+ Metrics.counter(ChangeStreamMetrics.class, "orphaned_new_partition_cleaned_count");
// -------------------
// Read change stream metrics
@@ -63,132 +69,111 @@ public class ChangeStreamMetrics implements Serializable {
* Counter for the total number of heartbeats identified during the execution of the Connector.
*/
public static final Counter HEARTBEAT_COUNT =
- Metrics.counter(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "heartbeat_count");
+ Metrics.counter(ChangeStreamMetrics.class, "heartbeat_count");
/**
* Counter for the total number of heartbeats identified during the execution of the Connector.
*/
public static final Counter CLOSESTREAM_COUNT =
- Metrics.counter(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "closestream_count");
+ Metrics.counter(ChangeStreamMetrics.class, "closestream_count");
/**
* Counter for the total number of ChangeStreamMutations that are initiated by users (not garbage
* collection) identified during the execution of the Connector.
*/
public static final Counter CHANGE_STREAM_MUTATION_USER_COUNT =
- Metrics.counter(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "change_stream_mutation_user_count");
+ Metrics.counter(ChangeStreamMetrics.class, "change_stream_mutation_user_count");
/**
* Counter for the total number of ChangeStreamMutations that are initiated by garbage collection
* (not user initiated) identified during the execution of the Connector.
*/
public static final Counter CHANGE_STREAM_MUTATION_GC_COUNT =
- Metrics.counter(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "change_stream_mutation_gc_count");
+ Metrics.counter(ChangeStreamMetrics.class, "change_stream_mutation_gc_count");
/** Distribution for measuring processing delay from commit timestamp. */
public static final Distribution PROCESSING_DELAY_FROM_COMMIT_TIMESTAMP =
- Metrics.distribution(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "processing_delay_from_commit_timestamp");
+ Metrics.distribution(ChangeStreamMetrics.class, "processing_delay_from_commit_timestamp");
/** Counter for the total number of active partitions being streamed. */
public static final Counter PARTITION_STREAM_COUNT =
- Metrics.counter(
- org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
- "partition_stream_count");
+ Metrics.counter(ChangeStreamMetrics.class, "partition_stream_count");
/**
- * Increments the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#LIST_PARTITIONS_COUNT} by
- * 1 if the metric is enabled.
+ * Increments the {@link ChangeStreamMetrics#LIST_PARTITIONS_COUNT} by 1 if the metric is enabled.
*/
public void incListPartitionsCount() {
inc(LIST_PARTITIONS_COUNT);
}
/**
- * Increments the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PARTITION_SPLIT_COUNT} by
- * 1 if the metric is enabled.
+ * Increments the {@link ChangeStreamMetrics#PARTITION_SPLIT_COUNT} by 1 if the metric is enabled.
*/
public void incPartitionSplitCount() {
inc(PARTITION_SPLIT_COUNT);
}
/**
- * Increments the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PARTITION_MERGE_COUNT} by
- * 1 if the metric is enabled.
+ * Increments the {@link ChangeStreamMetrics#PARTITION_MERGE_COUNT} by 1 if the metric is enabled.
*/
public void incPartitionMergeCount() {
inc(PARTITION_MERGE_COUNT);
}
- /**
- * Increments the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#HEARTBEAT_COUNT} by 1 if
- * the metric is enabled.
- */
+ /** Increments the {@link ChangeStreamMetrics#PARTITION_RECONCILED_WITH_TOKEN_COUNT} by 1. */
+ public void incPartitionReconciledWithTokenCount() {
+ inc(PARTITION_RECONCILED_WITH_TOKEN_COUNT);
+ }
+
+ /** Increments the {@link ChangeStreamMetrics#PARTITION_RECONCILED_WITHOUT_TOKEN_COUNT} by 1. */
+ public void incPartitionReconciledWithoutTokenCount() {
+ inc(PARTITION_RECONCILED_WITHOUT_TOKEN_COUNT);
+ }
+
+ /** Increments the {@link ChangeStreamMetrics#ORPHANED_NEW_PARTITION_CLEANED_COUNT} by 1. */
+ public void incOrphanedNewPartitionCleanedCount() {
+ inc(ORPHANED_NEW_PARTITION_CLEANED_COUNT);
+ }
+
+ /** Increments the {@link ChangeStreamMetrics#HEARTBEAT_COUNT} by 1 if the metric is enabled. */
public void incHeartbeatCount() {
inc(HEARTBEAT_COUNT);
}
- /**
- * Increments the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#CLOSESTREAM_COUNT} by 1
- * if the metric is enabled.
- */
+ /** Increments the {@link ChangeStreamMetrics#CLOSESTREAM_COUNT} by 1 if the metric is enabled. */
public void incClosestreamCount() {
inc(CLOSESTREAM_COUNT);
}
/**
- * Increments the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#CHANGE_STREAM_MUTATION_USER_COUNT}
- * by 1 if the metric is enabled.
+ * Increments the {@link ChangeStreamMetrics#CHANGE_STREAM_MUTATION_USER_COUNT} by 1 if the metric
+ * is enabled.
*/
public void incChangeStreamMutationUserCounter() {
inc(CHANGE_STREAM_MUTATION_USER_COUNT);
}
/**
- * Increments the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#CHANGE_STREAM_MUTATION_GC_COUNT}
- * by 1 if the metric is enabled.
+ * Increments the {@link ChangeStreamMetrics#CHANGE_STREAM_MUTATION_GC_COUNT} by 1 if the metric
+ * is enabled.
*/
public void incChangeStreamMutationGcCounter() {
inc(CHANGE_STREAM_MUTATION_GC_COUNT);
}
- /**
- * Increments the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PARTITION_STREAM_COUNT}
- * by 1.
- */
+ /** Increments the {@link ChangeStreamMetrics#PARTITION_STREAM_COUNT} by 1. */
public void incPartitionStreamCount() {
inc(PARTITION_STREAM_COUNT);
}
- /**
- * Decrements the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PARTITION_STREAM_COUNT}
- * by 1.
- */
+ /** Decrements the {@link ChangeStreamMetrics#PARTITION_STREAM_COUNT} by 1. */
public void decPartitionStreamCount() {
dec(PARTITION_STREAM_COUNT);
}
/**
* Adds measurement of an instance for the {@link
- * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PROCESSING_DELAY_FROM_COMMIT_TIMESTAMP}.
+ * ChangeStreamMetrics#PROCESSING_DELAY_FROM_COMMIT_TIMESTAMP}.
*/
public void updateProcessingDelayFromCommitTimestamp(long durationInMilli) {
update(PROCESSING_DELAY_FROM_COMMIT_TIMESTAMP, durationInMilli);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverter.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverter.java
index 4b664943f6a..bc58ef781ad 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverter.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverter.java
@@ -34,4 +34,8 @@ public class TimestampConverter {
public static long toSeconds(org.joda.time.Instant jodaInstant) {
return jodaInstant.getMillis() / 1000;
}
+
+ public static org.joda.time.Instant microsecondToInstant(long microsecond) {
+ return org.joda.time.Instant.ofEpochMilli(microsecond / 1_000L);
+ }
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/UniqueIdGenerator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/UniqueIdGenerator.java
index c5677f3feae..6999f47b6f3 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/UniqueIdGenerator.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/UniqueIdGenerator.java
@@ -26,14 +26,29 @@ import org.apache.beam.sdk.annotations.Internal;
public class UniqueIdGenerator {
private static final SecureRandom secureRandom = new SecureRandom();
+ /**
+ * Return a random base64 encoded 8 byte string. This is used to identify streamer of a specific
+ * partition. We expect there to be at most single digit duplicates of the same partition at any
+ * one time. The odd of collision if we generate <10 values per partition is less than 1 in 10^18.
+ *
+ * @return a random 8 byte string.
+ */
public static String getNextId() {
- byte[] bytes = new byte[256];
+ byte[] bytes = new byte[8];
secureRandom.nextBytes(bytes);
return Base64.getEncoder().encodeToString(bytes);
}
+ /**
+ * Return a random base64 encoded 8 byte string. This is used to identify a pipeline. Once a
+ * pipeline is complete, the metadata is left there and not cleaned up. It's possible, over the
+ * lifetime, there to be thousands and more pipelines in a single metadata table. The odds of
+ * collision if we expect 100,000 pipelines is less than 1 in 1 billion.
+ *
+ * @return a random 8 byte string.
+ */
public static String generateRowKeyPrefix() {
- byte[] bytes = new byte[50];
+ byte[] bytes = new byte[8];
secureRandom.nextBytes(bytes);
return Base64.getEncoder().encodeToString(bytes);
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java
index ac586ea147d..43623b5de16 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java
@@ -42,7 +42,9 @@ public class ActionFactory implements Serializable {
private transient ChangeStreamAction changeStreamAction;
private transient DetectNewPartitionsAction detectNewPartitionsAction;
+ private transient ProcessNewPartitionsAction processNewPartitionsAction;
private transient GenerateInitialPartitionsAction generateInitialPartitionsAction;
+ private transient ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction;
private transient ReadChangeStreamPartitionAction readChangeStreamPartitionAction;
/**
@@ -75,15 +77,31 @@ public class ActionFactory implements Serializable {
ChangeStreamMetrics metrics,
MetadataTableDao metadataTableDao,
@Nullable Instant endTime,
- GenerateInitialPartitionsAction generateInitialPartitionsAction) {
+ GenerateInitialPartitionsAction generateInitialPartitionsAction,
+ ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction,
+ ProcessNewPartitionsAction processNewPartitionsAction) {
if (detectNewPartitionsAction == null) {
detectNewPartitionsAction =
new DetectNewPartitionsAction(
- metrics, metadataTableDao, endTime, generateInitialPartitionsAction);
+ metrics,
+ metadataTableDao,
+ endTime,
+ generateInitialPartitionsAction,
+ resumeFromPreviousPipelineAction,
+ processNewPartitionsAction);
}
return detectNewPartitionsAction;
}
+ public synchronized ProcessNewPartitionsAction processNewPartitionsAction(
+ ChangeStreamMetrics metrics, MetadataTableDao metadataTableDao, @Nullable Instant endTime) {
+ if (processNewPartitionsAction == null) {
+ processNewPartitionsAction =
+ new ProcessNewPartitionsAction(metrics, metadataTableDao, endTime);
+ }
+ return processNewPartitionsAction;
+ }
+
/**
* Creates and returns a singleton instance of an action class for processing {@link
* org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.DetectNewPartitionsDoFn}
@@ -101,6 +119,19 @@ public class ActionFactory implements Serializable {
return generateInitialPartitionsAction;
}
+ public synchronized ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction(
+ ChangeStreamMetrics metrics,
+ MetadataTableDao metadataTableDao,
+ @Nullable Instant endTime,
+ ProcessNewPartitionsAction processNewPartitionsAction) {
+ if (resumeFromPreviousPipelineAction == null) {
+ resumeFromPreviousPipelineAction =
+ new ResumeFromPreviousPipelineAction(
+ metrics, metadataTableDao, endTime, processNewPartitionsAction);
+ }
+ return resumeFromPreviousPipelineAction;
+ }
+
/**
* Creates and returns a singleton instance of an action class for processing {@link
* org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.ReadChangeStreamPartitionDoFn}.
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java
index 7c002b8ede9..4c3f9c5a833 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java
@@ -136,7 +136,7 @@ public class ChangeStreamAction {
if (!tracker.tryClaim(streamProgress)) {
if (shouldDebug) {
LOG.info(
- "RCSP {}: Failed to claim heart beat tracker",
+ "RCSP {}: Checkpoint heart beat tracker",
formatByteStringRange(partitionRecord.getPartition()));
}
return Optional.of(DoFn.ProcessContinuation.stop());
@@ -167,7 +167,7 @@ public class ChangeStreamAction {
if (!tracker.tryClaim(streamProgress)) {
if (shouldDebug) {
LOG.info(
- "RCSP {}: Failed to claim close stream tracker",
+ "RCSP {}: Checkpoint close stream tracker",
formatByteStringRange(partitionRecord.getPartition()));
}
return Optional.of(DoFn.ProcessContinuation.stop());
@@ -192,7 +192,7 @@ public class ChangeStreamAction {
if (!tracker.tryClaim(streamProgress)) {
if (shouldDebug) {
LOG.info(
- "RCSP {}: Failed to claim data change tracker",
+ "RCSP {}: Checkpoint data change tracker",
formatByteStringRange(partitionRecord.getPartition()));
}
return Optional.of(DoFn.ProcessContinuation.stop());
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java
index 07ca7775436..d444f6a2dfd 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java
@@ -17,42 +17,35 @@
*/
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingAndOverlappingPartitionsFromKeySpace;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.isSuperset;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingPartitionsFromEntireKeySpace;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getOverlappingPartitions;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.partitionsToString;
-import com.google.api.gax.rpc.ServerStream;
-import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
-import com.google.cloud.bigtable.data.v2.models.Row;
-import com.google.cloud.bigtable.data.v2.models.RowCell;
-import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Set;
+import java.util.Optional;
import java.util.stream.Collectors;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
-import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
-import org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.DetectNewPartitionsState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.StreamPartitionWithWatermark;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler.OrphanedMetadataCleaner;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler.PartitionReconciler;
import org.apache.beam.sdk.io.range.OffsetRange;
-import org.apache.beam.sdk.transforms.DoFn.BundleFinalizer;
import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.slf4j.Logger;
@@ -74,26 +67,30 @@ public class DetectNewPartitionsAction {
private final ChangeStreamMetrics metrics;
private final MetadataTableDao metadataTableDao;
@Nullable private final Instant endTime;
+ private final ProcessNewPartitionsAction processNewPartitionsAction;
private final GenerateInitialPartitionsAction generateInitialPartitionsAction;
+ private final ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction;
private transient PartitionReconciler partitionReconciler;
+ private transient OrphanedMetadataCleaner orphanedMetadataCleaner;
public DetectNewPartitionsAction(
ChangeStreamMetrics metrics,
MetadataTableDao metadataTableDao,
@Nullable Instant endTime,
- GenerateInitialPartitionsAction generateInitialPartitionsAction) {
+ GenerateInitialPartitionsAction generateInitialPartitionsAction,
+ ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction,
+ ProcessNewPartitionsAction processNewPartitionsAction) {
this.metrics = metrics;
this.metadataTableDao = metadataTableDao;
this.endTime = endTime;
this.generateInitialPartitionsAction = generateInitialPartitionsAction;
+ this.resumeFromPreviousPipelineAction = resumeFromPreviousPipelineAction;
+ this.processNewPartitionsAction = processNewPartitionsAction;
}
/**
- * Periodically advances DetectNewPartition's (DNP) watermark based on the watermark of all the
- * partitions recorded in the metadata table. We don't advance DNP's watermark on every run to
- * "now" because of a possible inconsistent state. DNP's watermark is used to hold the entire
- * pipeline's watermark back.
+ * Get the new watermark based on the watermark of StreamPartitions and NewPartitions.
*
* <p>The low watermark is important to determine when to terminate the pipeline. During splits
* and merges, the watermark step may appear to be higher than it actually is. If a partition, at
@@ -110,171 +107,65 @@ public class DetectNewPartitionsAction {
* ensuring all the partitions exists in the metadata table in order to calculate the low
* watermark. It is possible that some partitions might be missing in between split and merges.
*
- * @param tracker restriction tracker to guide how frequently watermark should be advanced
- * @param watermarkEstimator watermark estimator to advance the watermark
+ * @param streamPartitionsWithWatermark restriction tracker to guide how frequently watermark
+ * should be advanced
+ * @param newPartitions watermark estimator to advance the watermark
+ * @return low watermark if possible, otherwise empty.
*/
- private void advanceWatermark(
- RestrictionTracker<OffsetRange, Long> tracker,
- ManualWatermarkEstimator<Instant> watermarkEstimator)
- throws InvalidProtocolBufferException {
- // We currently choose to update the watermark every 10 runs. We want to choose a number that is
- // frequent, so the watermark isn't lagged behind too far. Also not too frequent so we do not
- // overload the table with full table scans.
- if (tracker.currentRestriction().getFrom() % 10 == 0) {
- // Get partitions with a watermark set but skip rows w a lock and no watermark yet
- ServerStream<Row> rows = metadataTableDao.readFromMdTableStreamPartitionsWithWatermark();
- List<ByteStringRange> partitions = new ArrayList<>();
- HashMap<ByteStringRange, Instant> slowPartitions = new HashMap<>();
- Instant lowWatermark = Instant.ofEpochMilli(Long.MAX_VALUE);
- for (Row row : rows) {
- Instant watermark = MetadataTableEncoder.parseWatermarkFromRow(row);
- if (watermark == null) {
- continue;
- }
- // Update low watermark if watermark < low watermark.
- if (watermark.compareTo(lowWatermark) < 0) {
- lowWatermark = watermark;
- }
- ByteStringRange partition =
- metadataTableDao.convertStreamPartitionRowKeyToPartition(row.getKey());
- partitions.add(partition);
- if (watermark.plus(DEBUG_WATERMARK_DELAY).isBeforeNow()) {
- slowPartitions.put(partition, watermark);
- }
- }
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
- if (missingAndOverlappingPartitions.isEmpty()) {
- watermarkEstimator.setWatermark(lowWatermark);
- LOG.info("DNP: Updating watermark: " + watermarkEstimator.currentWatermark());
- } else {
- LOG.warn(
- "DNP: Could not update watermark because missing {}",
- partitionsToString(missingAndOverlappingPartitions));
+ private Optional<Instant> getNewWatermark(
+ List<StreamPartitionWithWatermark> streamPartitionsWithWatermark,
+ List<NewPartition> newPartitions) {
+ // Get partitions with a watermark set but skip rows w a lock and no watermark yet
+ List<StreamPartitionWithWatermark> slowPartitions = new ArrayList<>();
+ Instant lowWatermark = Instant.ofEpochMilli(Long.MAX_VALUE);
+
+ List<ByteStringRange> partitions = new ArrayList<>();
+ for (StreamPartitionWithWatermark streamPartitionWithWatermark :
+ streamPartitionsWithWatermark) {
+ if (streamPartitionWithWatermark.getWatermark().plus(DEBUG_WATERMARK_DELAY).isBeforeNow()) {
+ slowPartitions.add(streamPartitionWithWatermark);
}
- if (!slowPartitions.isEmpty()) {
- LOG.warn(
- "DNP: Watermark is being held back by the following partitions: {}",
- slowPartitions.entrySet().stream()
- .map(
- e ->
- ByteStringRangeHelper.formatByteStringRange(e.getKey())
- + " => "
- + e.getValue())
- .collect(Collectors.joining(", ", "{", "}")));
+ if (streamPartitionWithWatermark.getWatermark().compareTo(lowWatermark) < 0) {
+ lowWatermark = streamPartitionWithWatermark.getWatermark();
}
- partitionReconciler.addMissingPartitions(missingAndOverlappingPartitions);
- }
- }
-
- /**
- * Process a single new partition. New partition resulting from split and merges need to be
- * outputted to be streamed. Regardless if it's a split or a merge, we have the same verification
- * process in order to ensure the new partition can actually be streamed.
- *
- * <p>When a parent partition splits, it receives two or more new partitions. It will write a new
- * row, with the new row ranges as row key, for each new partition. These new partitions can be
- * immediately streamed.
- *
- * <p>The complicated scenario is merges. Two or more parent partitions will merge into one new
- * partition. Each parent partition receives the same new partition (row range) but each parent
- * partition will have a different continuation token. The parent partitions will all write to the
- * same row key form by the new row range. Each parent will record its continuation token, and
- * watermark. Parent partitions may not receive the message to stop at the same time. So when we
- * try to process the new partition, we need to ensure that all the parent partitions have stopped
- * and recorded their metadata table. We do so by verifying that the row ranges of the parents
- * covers a contiguous block of row range that is same or wider (superset) of the new row range.
- *
- * <p>For example, partition1, A-B, and partition2, B-C, merges into partition3, A-C.
- *
- * <ol>
- * <li>p1 writes to row A-C in metadata table
- * <li>processNewPartition process A-C seeing that only A-B has been recorded and A-B does not
- * cover A-C. Do Nothing
- * <li>p2 writes to row A-C in metadata table
- * <li>processNewPartition process A-C again, seeing that A-B and B-C has been recorded and
- * outputs new partition A-C to be streamed.
- * </ol>
- *
- * <p>Note that, the algorithm to verify if a merge is valid, also correctly verifies if a split
- * is valid. A split is immediately valid as long as the row exists because there's only one
- * parent that needs to write to that row.
- *
- * @param row new partition to be processed
- * @param receiver to output new partitions
- * @param recordedPartitionRecords add the partition if it's been processed
- * @throws InvalidProtocolBufferException if partition can't be converted from row key.
- */
- private void processNewPartition(
- Row row, OutputReceiver<PartitionRecord> receiver, List<ByteString> recordedPartitionRecords)
- throws Exception {
- ByteStringRange partition = metadataTableDao.convertNewPartitionRowKeyToPartition(row.getKey());
-
- partitionReconciler.addNewPartition(partition, row.getKey());
-
- // Ensure all parent partitions have stopped and updated the metadata table with the new
- // partition continuation token.
- List<ByteStringRange> parentPartitions = new ArrayList<>();
- for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_PARENT_PARTITIONS)) {
- ByteStringRange parentPartition = ByteStringRange.toByteStringRange(cell.getQualifier());
- parentPartitions.add(parentPartition);
+ partitions.add(streamPartitionWithWatermark.getPartition());
}
- if (!isSuperset(parentPartitions, partition)) {
+ if (!slowPartitions.isEmpty()) {
LOG.warn(
- "DNP: New partition: {} does not have all the parents {}",
- ByteStringRangeHelper.formatByteStringRange(partition),
- partitionsToString(parentPartitions));
- return;
- }
-
- // Capture all the initial continuation tokens to request for the new partition.
- ImmutableList.Builder<ChangeStreamContinuationToken> changeStreamContinuationTokensBuilder =
- ImmutableList.builder();
- for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_INITIAL_TOKEN)) {
- ChangeStreamContinuationToken changeStreamContinuationToken =
- ChangeStreamContinuationToken.fromByteString(cell.getQualifier());
- changeStreamContinuationTokensBuilder.add(changeStreamContinuationToken);
+ "DNP: Updating watermark is held back by {} partitions : {}",
+ slowPartitions.size(),
+ slowPartitions.stream()
+ .map(e -> formatByteStringRange(e.getPartition()) + " => " + e.getWatermark())
+ .collect(Collectors.joining(", ", "{", "}")));
}
- ImmutableList<ChangeStreamContinuationToken> changeStreamContinuationTokens =
- changeStreamContinuationTokensBuilder.build();
- // If parent and continuation token count are not the same, it's possible the same parent wrote
- // multiple continuation tokens or there's an inconsistent state.
- if (parentPartitions.size() != changeStreamContinuationTokens.size()) {
+ // Only added StreamPartitions so far, check if there are any overlapping StreamPartitions. If
+ // so, something is wrong, we should stop.
+ List<ByteStringRange> overlappingStreamPartitions = getOverlappingPartitions(partitions);
+ if (!overlappingStreamPartitions.isEmpty()) {
LOG.warn(
- "DNP: New partition {} parent partitions count {} != continuation token count {}",
- ByteStringRangeHelper.formatByteStringRange(partition),
- parentPartitions.size(),
- changeStreamContinuationTokens.size());
+ "DNP: Updating watermark failed due to overlapping: {}",
+ partitionsToString(overlappingStreamPartitions));
+ return Optional.empty();
}
- // Capture all the low watermark and calculate the low watermark among all the parents.
- List<Long> parentLowWatermark = new ArrayList<>();
- for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS)) {
- parentLowWatermark.add(Longs.fromByteArray(cell.getValue().toByteArray()));
+ for (NewPartition newPartition : newPartitions) {
+ partitions.addAll(newPartition.getParentPartitions());
+ if (newPartition.getLowWatermark().compareTo(lowWatermark) < 0) {
+ lowWatermark = newPartition.getLowWatermark();
+ }
}
- Long lowWatermarkMilli = Collections.min(parentLowWatermark);
- Instant lowWatermark = Instant.ofEpochMilli(lowWatermarkMilli);
- String uid = UniqueIdGenerator.getNextId();
- PartitionRecord partitionRecord =
- new PartitionRecord(partition, changeStreamContinuationTokens, uid, lowWatermark, endTime);
- if (parentPartitions.size() > 1) {
- metrics.incPartitionMergeCount();
- } else {
- metrics.incPartitionSplitCount();
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ if (missingPartitions.isEmpty()) {
+ LOG.info("DNP: Updating watermark: " + lowWatermark);
+ return Optional.of(lowWatermark);
}
- LOG.info(
- "DNP: Split/Merge {} into {}",
- parentPartitions.stream()
- .map(ByteStringRangeHelper::formatByteStringRange)
- .collect(Collectors.joining(",", "{", "}")),
- ByteStringRangeHelper.formatByteStringRange(partition));
- // We are outputting elements with timestamp of 0 to prevent reliance on event time. This limits
- // the ability to window on commit time of any data changes. It is still possible to window on
- // processing time.
- receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
- recordedPartitionRecords.add(row.getKey());
+ LOG.warn(
+ "DNP: Updating watermark failed due to missing {} partitions : {}.",
+ missingPartitions.size(),
+ partitionsToString(missingPartitions));
+ return Optional.empty();
}
/**
@@ -283,69 +174,54 @@ public class DetectNewPartitionsAction {
*
* @param receiver used to output reconciled partitions
* @param watermarkEstimator read the low watermark for all partitions
- * @param startTime startTime of the pipeline
- * @param recordedPartitionRecords the list of row keys that needs to be cleaned up
*/
private void processReconcilerPartitions(
OutputReceiver<PartitionRecord> receiver,
ManualWatermarkEstimator<Instant> watermarkEstimator,
- Instant startTime,
- List<ByteString> recordedPartitionRecords) {
- for (HashMap.Entry<ByteStringRange, Set<ByteString>> partitionToReconcile :
- partitionReconciler.getPartitionsToReconcile().entrySet()) {
- String uid = UniqueIdGenerator.getNextId();
+ Instant startTime) {
+ for (PartitionRecord reconciledPartition :
+ partitionReconciler.getPartitionsToReconcile(
+ watermarkEstimator.currentWatermark(), startTime)) {
+ // Set the missing fields.
+ reconciledPartition.setUuid(UniqueIdGenerator.getNextId());
+ reconciledPartition.setEndTime(endTime);
- // When we reconcile, we start from 1h prior to effectively eliminate the possibility of
- // missing data.
- Instant reconciledTime =
- watermarkEstimator.currentWatermark().minus(Duration.standardMinutes(60));
- if (reconciledTime.compareTo(startTime) < 0) {
- reconciledTime = startTime;
+ for (NewPartition parentPartition : reconciledPartition.getParentPartitions()) {
+ metadataTableDao.markNewPartitionForDeletion(parentPartition);
}
+ receiver.outputWithTimestamp(reconciledPartition, Instant.EPOCH);
+ LOG.warn("DNP: Reconciling missing partition: {}", reconciledPartition);
+ }
+ }
- PartitionRecord partitionRecord =
- new PartitionRecord(
- partitionToReconcile.getKey(), reconciledTime, uid, reconciledTime, endTime);
- receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
- recordedPartitionRecords.addAll(partitionToReconcile.getValue());
- LOG.warn(
- "DNP: Reconciling missing partition: {} and cleaning up rows {}",
- partitionRecord,
- partitionToReconcile.getValue().stream()
- .map(
- rowKey -> {
- try {
- return ByteStringRangeHelper.formatByteStringRange(
- metadataTableDao.convertNewPartitionRowKeyToPartition(rowKey));
- } catch (InvalidProtocolBufferException exception) {
- return rowKey.toStringUtf8();
- }
- })
- .collect(Collectors.joining(", ", "{", "}")));
+ private void cleanUpOrphanedMetadata() {
+ for (NewPartition newPartitionToClean : orphanedMetadataCleaner.getOrphanedNewPartitions()) {
+ metrics.incOrphanedNewPartitionCleanedCount();
+ metadataTableDao.markNewPartitionForDeletion(newPartitionToClean);
+ metadataTableDao.deleteNewPartition(newPartitionToClean);
}
}
/**
- * After processing new partitions and if it was outputted successfully, we need to clean up the
- * metadata table so that we don't try to process the same new partition again.
+ * Return true if watermark should be updated.
+ *
+ * <p>We update the watermark on even count after 10s since last update.
*
- * <p>This is performed at a best effort basis. If some clean up is unsuccessful, the new
- * partition will be processed and output again in subsequent runs. This is OK because the DoFn
- * that stream partitions performs locking to prevent multiple DoFn from streaming the same
- * partition.
+ * <p>We chose to only update on even count, so we don't perform a full table scan on every DNP
+ * run if DNP taking longer than 10s between each run.
*
- * @param bundleFinalizer finalizer that registered the callback to perform the cleanup
- * @param recordedPartitionRecords the list of row keys that needs to be cleaned up
+ * <p>We set a minimum of 10s to minimize frequency of full table scan. This means the watermark
+ * will always lag behind by 10s. We didn't choose a larger minimum frequency because that would
+ * increase the minimum watermark delay.
*/
- private void cleanUpAfterCommit(
- BundleFinalizer bundleFinalizer, List<ByteString> recordedPartitionRecords) {
- bundleFinalizer.afterBundleCommit(
- Instant.ofEpochMilli(Long.MAX_VALUE),
- () -> {
- for (ByteString rowKey : recordedPartitionRecords) {
- metadataTableDao.deleteRowKey(rowKey);
- }
- });
+ private boolean shouldUpdateWatermark(
+ long count, @Nullable DetectNewPartitionsState detectNewPartitionsState) {
+ return count % 2 == 0
+ && (detectNewPartitionsState == null
+ || detectNewPartitionsState
+ .getWatermarkLastUpdated()
+ .plus(Duration.standardSeconds(10))
+ .isBeforeNow());
}
/**
@@ -366,7 +242,6 @@ public class DetectNewPartitionsAction {
* @param receiver output new partitions
* @param watermarkEstimator update watermark that is a representation of the low watermark of the
* entire beam pipeline
- * @param bundleFinalizer perform after bundle output actions to clean up metadata table
* @return {@link ProcessContinuation#resume()} with 1-second delay if the stream continues,
* otherwise {@link ProcessContinuation#stop()}
* @throws InvalidProtocolBufferException if failing to process new partitions
@@ -376,23 +251,48 @@ public class DetectNewPartitionsAction {
RestrictionTracker<OffsetRange, Long> tracker,
OutputReceiver<PartitionRecord> receiver,
ManualWatermarkEstimator<Instant> watermarkEstimator,
- BundleFinalizer bundleFinalizer,
- Instant startTime)
+ InitialPipelineState initialPipelineState)
throws Exception {
LOG.debug("DNP: Watermark: " + watermarkEstimator.getState());
LOG.debug("DNP: CurrentTracker: " + tracker.currentRestriction().getFrom());
if (tracker.currentRestriction().getFrom() == 0L) {
- return generateInitialPartitionsAction.run(receiver, tracker, watermarkEstimator, startTime);
+ if (!tracker.tryClaim(0L)) {
+ LOG.error(
+ "Could not claim initial DetectNewPartition restriction. No partitions are outputted.");
+ return ProcessContinuation.stop();
+ }
+ watermarkEstimator.setWatermark(initialPipelineState.getStartTime());
+ if (initialPipelineState.isResume()) {
+ resumeFromPreviousPipelineAction.run(receiver);
+ } else {
+ generateInitialPartitionsAction.run(receiver, initialPipelineState.getStartTime());
+ }
+ return ProcessContinuation.resume();
}
// Create a new partition reconciler every run to reset the state each time.
- partitionReconciler = new PartitionReconciler(metadataTableDao);
+ partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
+ orphanedMetadataCleaner = new OrphanedMetadataCleaner();
- advanceWatermark(tracker, watermarkEstimator);
+ // Calculating the new value of watermark is a resource intensive process. We have to do a full
+ // scan of the metadata table and then ensure we're not missing partitions and then calculate
+ // the low watermark. This is usually a fairly fast process even with thousands of partitions.
+ // However, sometimes this may take so long that the runner checkpoints before the watermark is
+ // calculated. Because the checkpoint takes place before tryClaim, this forces the DoFn to
+ // restart, wasting the resources spent calculating the watermark. On restart, we will try to
+ // calculate the watermark again. The problem causing the slow watermark calculation can persist
+ // leading to a crash loop. In order to ensure we persist the calculated watermark, we calculate
+ // the watermark after successful tryClaim. Then we write to the metadata table the new
+ // watermark. On the start of each run we read the watermark and update the DoFn's watermark.
+ DetectNewPartitionsState detectNewPartitionsState =
+ metadataTableDao.readDetectNewPartitionsState();
+ if (detectNewPartitionsState != null) {
+ watermarkEstimator.setWatermark(detectNewPartitionsState.getWatermark());
+ }
// Terminate if endTime <= watermark that means all partitions have read up to or beyond
// watermark. We no longer need to manage splits and merges, we can terminate.
- if (endTime != null && endTime.isBefore(watermarkEstimator.currentWatermark())) {
+ if (endTime != null && !watermarkEstimator.currentWatermark().isBefore(endTime)) {
tracker.tryClaim(tracker.currentRestriction().getTo());
return ProcessContinuation.stop();
}
@@ -402,17 +302,51 @@ public class DetectNewPartitionsAction {
return ProcessContinuation.stop();
}
- List<ByteString> recordedPartitionRecords = new ArrayList<>();
+ // Read StreamPartitions to calculate watermark.
+ List<StreamPartitionWithWatermark> streamPartitionsWithWatermark = null;
+ if (shouldUpdateWatermark(tracker.currentRestriction().getFrom(), detectNewPartitionsState)) {
+ streamPartitionsWithWatermark = metadataTableDao.readStreamPartitionsWithWatermark();
+ }
- ServerStream<Row> rows = metadataTableDao.readNewPartitions();
- for (Row row : rows) {
- processNewPartition(row, receiver, recordedPartitionRecords);
+ // Process NewPartitions and track the ones successfully outputted.
+ List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+ List<ByteStringRange> outputtedNewPartitions = new ArrayList<>();
+ for (NewPartition newPartition : newPartitions) {
+ if (processNewPartitionsAction.processNewPartition(newPartition, receiver)) {
+ outputtedNewPartitions.add(newPartition.getPartition());
+ } else {
+ partitionReconciler.addIncompleteNewPartitions(newPartition);
+ orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition);
+ }
}
- processReconcilerPartitions(receiver, watermarkEstimator, startTime, recordedPartitionRecords);
+ // Process the watermark using read StreamPartitions and NewPartitions.
+ if (streamPartitionsWithWatermark != null) {
+ Optional<Instant> maybeWatermark =
+ getNewWatermark(streamPartitionsWithWatermark, newPartitions);
+ maybeWatermark.ifPresent(metadataTableDao::updateDetectNewPartitionWatermark);
+ // Using NewPartitions and StreamPartitions, evaluate partitions that are possibly not being
+ // streamed. This isn't perfect because there may be partitions moving between
+ // StreamPartitions and NewPartitions while scanning the metadata table. Also, this does not
+ // include NewPartitions marked as deleted from a previous DNP run not yet processed by RCSP.
+ List<ByteStringRange> existingPartitions =
+ streamPartitionsWithWatermark.stream()
+ .map(StreamPartitionWithWatermark::getPartition)
+ .collect(Collectors.toList());
+ existingPartitions.addAll(outputtedNewPartitions);
+ List<ByteStringRange> missingStreamPartitions =
+ getMissingPartitionsFromEntireKeySpace(existingPartitions);
+ orphanedMetadataCleaner.addMissingPartitions(missingStreamPartitions);
+ partitionReconciler.addMissingPartitions(missingStreamPartitions);
+ }
- cleanUpAfterCommit(bundleFinalizer, recordedPartitionRecords);
+ // Only start reconciling after the pipeline has been running for a while.
+ if (tracker.currentRestriction().getFrom() > 50) {
+ processReconcilerPartitions(
+ receiver, watermarkEstimator, initialPipelineState.getStartTime());
+ cleanUpOrphanedMetadata();
+ }
- return ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1));
+ return ProcessContinuation.resume().withResumeDelay(Duration.millis(100));
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsAction.java
index d5a1fc6b74e..15c030f1da3 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsAction.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsAction.java
@@ -18,6 +18,7 @@
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.beam.sdk.annotations.Internal;
@@ -25,14 +26,8 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
-import org.apache.beam.sdk.io.range.OffsetRange;
import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
-import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
-import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.joda.time.Instant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Class to generate first set of outputs for {@link
@@ -40,8 +35,6 @@ import org.slf4j.LoggerFactory;
*/
@Internal
public class GenerateInitialPartitionsAction {
- private static final Logger LOG = LoggerFactory.getLogger(GenerateInitialPartitionsAction.class);
-
private final ChangeStreamMetrics metrics;
private final ChangeStreamDao changeStreamDao;
@Nullable private final Instant endTime;
@@ -56,35 +49,20 @@ public class GenerateInitialPartitionsAction {
/**
* The very first step of the pipeline when there are no partitions being streamed yet. We want to
* get an initial list of partitions to stream and output them.
- *
- * @return {@link ProcessContinuation#resume()} if the stream continues, otherwise {@link
- * ProcessContinuation#stop()}
*/
- public ProcessContinuation run(
- OutputReceiver<PartitionRecord> receiver,
- RestrictionTracker<OffsetRange, Long> tracker,
- ManualWatermarkEstimator<Instant> watermarkEstimator,
- Instant startTime) {
- if (!tracker.tryClaim(0L)) {
- LOG.error(
- "Could not claim initial DetectNewPartition restriction. No partitions are outputted.");
- return ProcessContinuation.stop();
- }
+ public void run(OutputReceiver<PartitionRecord> receiver, Instant startTime) {
List<ByteStringRange> streamPartitions =
changeStreamDao.generateInitialChangeStreamPartitions();
-
- watermarkEstimator.setWatermark(startTime);
-
for (ByteStringRange partition : streamPartitions) {
metrics.incListPartitionsCount();
String uid = UniqueIdGenerator.getNextId();
PartitionRecord partitionRecord =
- new PartitionRecord(partition, startTime, uid, startTime, endTime);
+ new PartitionRecord(
+ partition, startTime, uid, startTime, Collections.emptyList(), endTime);
// We are outputting elements with timestamp of 0 to prevent reliance on event time. This
// limits the ability to window on commit time of any data changes. It is still possible to
// window on processing time.
receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
}
- return ProcessContinuation.resume();
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ProcessNewPartitionsAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ProcessNewPartitionsAction.java
new file mode 100644
index 00000000000..e16054a5107
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ProcessNewPartitionsAction.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
+
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.coverSameKeySpace;
+
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProcessNewPartitionsAction {
+ private static final Logger LOG = LoggerFactory.getLogger(ProcessNewPartitionsAction.class);
+
+ private final ChangeStreamMetrics metrics;
+ private final MetadataTableDao metadataTableDao;
+ @Nullable private final Instant endTime;
+
+ public ProcessNewPartitionsAction(
+ ChangeStreamMetrics metrics, MetadataTableDao metadataTableDao, @Nullable Instant endTime) {
+ this.metrics = metrics;
+ this.metadataTableDao = metadataTableDao;
+ this.endTime = endTime;
+ }
+
+ /**
+ * Process a single new partition. New partition resulting from split and merges need to be
+ * outputted to be streamed. Regardless if it's a split or a merge, we have the same verification
+ * process in order to ensure the new partition can actually be streamed.
+ *
+ * <p>When a parent partition splits, it receives two or more new partitions. It will write a new
+ * row, with the new row ranges as row key, for each new partition. These new partitions can be
+ * immediately streamed.
+ *
+ * <p>The complicated scenario is merges. Two or more parent partitions will merge into one new
+ * partition. Each parent partition receives the same new partition (row range) but each parent
+ * partition will have a different continuation token. The parent partitions will all write to the
+ * same row key form by the new row range. Each parent will record its continuation token, and
+ * watermark. Parent partitions may not receive the message to stop at the same time. So when we
+ * try to process the new partition, we need to ensure that all the parent partitions have stopped
+ * and recorded their metadata table. We do so by verifying that the row ranges of the parents
+ * covers a contiguous block of row range that is same as the new row range.
+ *
+ * <p>For example, partition1, A-B, and partition2, B-C, merges into partition3, A-C.
+ *
+ * <ol>
+ * <li>p1 writes to row A-C in metadata table
+ * <li>processNewPartition process A-C seeing that only A-B has been recorded and A-B does not
+ * cover A-C. Do Nothing
+ * <li>p2 writes to row A-C in metadata table
+ * <li>processNewPartition process A-C again, seeing that A-B and B-C has been recorded and
+ * outputs new partition A-C to be streamed.
+ * </ol>
+ *
+ * <p>Note that, the algorithm to verify if a merge is valid, also correctly verifies if a split
+ * is valid. A split is immediately valid as long as the row exists because there's only one
+ * parent that needs to write to that row.
+ *
+ * @param newPartition new partition to be processed
+ * @param receiver to output new partitions
+ */
+ public boolean processNewPartition(
+ NewPartition newPartition, OutputReceiver<PartitionRecord> receiver) {
+ List<ByteStringRange> parentPartitions = newPartition.getParentPartitions();
+ if (!coverSameKeySpace(parentPartitions, newPartition.getPartition())) {
+ LOG.warn("DNP: New partition does not have all the parents: {}", newPartition);
+ return false;
+ }
+
+ String uuid = UniqueIdGenerator.getNextId();
+ PartitionRecord partitionRecord =
+ new PartitionRecord(
+ newPartition.getPartition(),
+ newPartition.getChangeStreamContinuationTokens(),
+ uuid,
+ newPartition.getLowWatermark(),
+ Collections.singletonList(newPartition),
+ endTime);
+ if (parentPartitions.size() > 1) {
+ metrics.incPartitionMergeCount();
+ } else {
+ metrics.incPartitionSplitCount();
+ }
+ LOG.info("DNP: Outputting new partition: {}", partitionRecord);
+ metadataTableDao.markNewPartitionForDeletion(newPartition);
+ // We are outputting elements with timestamp of 0 to prevent reliance on event time. This limits
+ // the ability to window on commit time of any data changes. It is still possible to window on
+ // processing time.
+ receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
+ return true;
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java
index 32262a7d35a..2270cbd08ac 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java
@@ -17,8 +17,8 @@
*/
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.coverSameKeySpace;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.isSuperset;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.partitionsToString;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamContinuationTokenHelper.getTokenWithCorrectPartition;
@@ -32,6 +32,7 @@ import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
import com.google.protobuf.ByteString;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Optional;
import org.apache.beam.sdk.annotations.Internal;
@@ -39,6 +40,7 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.DetectNewPartitionsDoFn;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress;
import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
@@ -138,48 +140,76 @@ public class ReadChangeStreamPartitionAction {
if (shouldDebug) {
LOG.info(
- "RCSP: Partition: "
+ "RCSP {}: Partition: "
+ partitionRecord
+ "\n Watermark: "
+ watermarkEstimator.getState()
+ "\n RestrictionTracker: "
- + tracker.currentRestriction());
+ + tracker.currentRestriction(),
+ formatByteStringRange(partitionRecord.getPartition()));
}
// Lock the partition
- if (!metadataTableDao.lockPartition(
- partitionRecord.getPartition(), partitionRecord.getUuid())) {
- LOG.info(
- "RCSP: Could not acquire lock for partition: {}, with uid: {}, because this is a "
- + "duplicate and another worker is working on this partition already.",
+ if (tracker.currentRestriction().isEmpty()) {
+ boolean lockedPartition = metadataTableDao.lockAndRecordPartition(partitionRecord);
+ // Clean up NewPartition on the first run regardless of locking result. If locking fails it
+ // means this partition is being streamed, then cleaning up NewPartitions avoids lingering
+ // NewPartitions.
+ for (NewPartition newPartition : partitionRecord.getParentPartitions()) {
+ metadataTableDao.deleteNewPartition(newPartition);
+ }
+ if (!lockedPartition) {
+ LOG.info(
+ "RCSP {} : Could not acquire lock with uid: {}, because this is a "
+ + "duplicate and another worker is working on this partition already.",
+ formatByteStringRange(partitionRecord.getPartition()),
+ partitionRecord.getUuid());
+ StreamProgress streamProgress = new StreamProgress();
+ streamProgress.setFailToLock(true);
+ metrics.decPartitionStreamCount();
+ tracker.tryClaim(streamProgress);
+ return ProcessContinuation.stop();
+ }
+ } else if (tracker.currentRestriction().getCloseStream() == null
+ && !metadataTableDao.doHoldLock(
+ partitionRecord.getPartition(), partitionRecord.getUuid())) {
+ // We only verify the lock if we are not holding CloseStream because if this is a retry of
+ // CloseStream we might have already cleaned up the lock in a previous attempt.
+ // Failed correctness check on this worker holds the lock on this partition. This shouldn't
+ // fail because there's a restriction tracker which means this worker has already acquired the
+ // lock and once it has acquired the lock it shouldn't fail the lock check.
+ LOG.warn(
+ "RCSP {} : Subsequent run that doesn't hold the lock {}. This is not unexpected and "
+ + "should probably be reviewed.",
formatByteStringRange(partitionRecord.getPartition()),
partitionRecord.getUuid());
StreamProgress streamProgress = new StreamProgress();
streamProgress.setFailToLock(true);
metrics.decPartitionStreamCount();
- if (!tracker.tryClaim(streamProgress)) {
- LOG.debug("RCSP: Failed to claim tracker after failing to lock partition.");
- return ProcessContinuation.stop();
- }
+ tracker.tryClaim(streamProgress);
return ProcessContinuation.stop();
}
// Process CloseStream if it exists
CloseStream closeStream = tracker.currentRestriction().getCloseStream();
if (closeStream != null) {
+ LOG.debug("RCSP: Processing CloseStream");
+ metrics.decPartitionStreamCount();
if (closeStream.getStatus().getCode() == Status.Code.OK) {
// We need to update watermark here. We're terminating this stream because we have reached
// endTime. Instant.now is greater or equal to endTime. The goal here is
// DNP will need to know this stream has passed the endTime so DNP can eventually terminate.
- watermarkEstimator.setWatermark(Instant.now());
+ Instant terminatingWatermark = Instant.ofEpochMilli(Long.MAX_VALUE);
+ Instant endTime = partitionRecord.getEndTime();
+ if (endTime != null) {
+ terminatingWatermark = endTime;
+ }
+ watermarkEstimator.setWatermark(terminatingWatermark);
metadataTableDao.updateWatermark(
- partitionRecord.getPartition(),
- watermarkEstimator.currentWatermark(),
- tracker.currentRestriction().getCurrentToken());
+ partitionRecord.getPartition(), watermarkEstimator.currentWatermark(), null);
LOG.info(
"RCSP {}: Reached end time, terminating...",
formatByteStringRange(partitionRecord.getPartition()));
- metrics.decPartitionStreamCount();
return ProcessContinuation.stop();
}
if (closeStream.getStatus().getCode() != Status.Code.OUT_OF_RANGE) {
@@ -187,14 +217,27 @@ public class ReadChangeStreamPartitionAction {
"RCSP {}: Reached unexpected terminal state: {}",
formatByteStringRange(partitionRecord.getPartition()),
closeStream.getStatus());
- metrics.decPartitionStreamCount();
return ProcessContinuation.stop();
}
- // The partitions in the continuation tokens should be a superset of this partition.
- // If there's only 1 token, then the token's partition should be a superset of this partition.
- // If there are more than 1 tokens, then the tokens should form a continuous row range that is
- // a superset of this partition.
+ // Release the lock only if the uuid matches. In normal operation this doesn't change
+ // anything. However, it's possible for this RCSP to crash while processing CloseStream but
+ // after the side effects of writing the new partitions to the metadata table. New partitions
+ // can be created while this RCSP restarts from the previous checkpoint and processes the
+ // CloseStream again. In certain race scenarios the child partitions may merge back to this
+ // partition, but as a new RCSP. The new partition (same as this partition) would write the
+ // exact same content to the metadata table but with a different uuid. We don't want to
+ // accidentally delete the StreamPartition because it now belongs to the new RCSP.
+ // If the uuid is the same (meaning this race scenario did not take place) we release the lock
+ // and mark the StreamPartition to be deleted, so we can delete it after we have written the
+ // NewPartitions.
+ metadataTableDao.releaseStreamPartitionLockForDeletion(
+ partitionRecord.getPartition(), partitionRecord.getUuid());
+ // The partitions in the continuation tokens must cover the same key space as this partition.
+ // If there's only 1 token, then the token's partition is equals to this partition.
+ // If there are more than 1 tokens, then the tokens form a continuous row range equals to this
+ // partition.
List<ByteStringRange> childPartitions = new ArrayList<>();
+ List<ByteStringRange> tokenPartitions = new ArrayList<>();
// Check if NewPartitions field exists, if not we default to using just the
// ChangeStreamContinuationTokens.
boolean useNewPartitionsField =
@@ -212,8 +255,10 @@ public class ReadChangeStreamPartitionAction {
getTokenWithCorrectPartition(
partitionRecord.getPartition(),
closeStream.getChangeStreamContinuationTokens().get(i));
+ tokenPartitions.add(token.getPartition());
metadataTableDao.writeNewPartition(
- childPartition, token, partitionRecord.getPartition(), watermarkEstimator.getState());
+ new NewPartition(
+ childPartition, Collections.singletonList(token), watermarkEstimator.getState()));
}
if (shouldDebug) {
LOG.info(
@@ -221,14 +266,16 @@ public class ReadChangeStreamPartitionAction {
formatByteStringRange(partitionRecord.getPartition()),
partitionsToString(childPartitions));
}
- if (!isSuperset(childPartitions, partitionRecord.getPartition())) {
+ if (!coverSameKeySpace(tokenPartitions, partitionRecord.getPartition())) {
LOG.warn(
- "RCSP {}: CloseStream has child partition(s) {} that doesn't cover the keyspace",
+ "RCSP {}: CloseStream has tokens {} that don't cover the entire keyspace",
formatByteStringRange(partitionRecord.getPartition()),
- partitionsToString(childPartitions));
+ partitionsToString(tokenPartitions));
}
+ // Perform the real cleanup. This step is no op if the race mentioned above occurs (splits and
+ // merges results back to this partition again) because when we register the "new" partition,
+ // we unset the deletion bit.
metadataTableDao.deleteStreamPartitionRow(partitionRecord.getPartition());
- metrics.decPartitionStreamCount();
return ProcessContinuation.stop();
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineAction.java
new file mode 100644
index 00000000000..79d6ae90a25
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineAction.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.HashMap;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ResumeFromPreviousPipelineAction {
+ private static final Logger LOG = LoggerFactory.getLogger(ResumeFromPreviousPipelineAction.class);
+
+ private final ChangeStreamMetrics metrics;
+ private final MetadataTableDao metadataTableDao;
+ @Nullable private final Instant endTime;
+ private final ProcessNewPartitionsAction processNewPartitionsAction;
+
+ public ResumeFromPreviousPipelineAction(
+ ChangeStreamMetrics metrics,
+ MetadataTableDao metadataTableDao,
+ @Nullable Instant endTime,
+ ProcessNewPartitionsAction processNewPartitionsAction) {
+ this.metrics = metrics;
+ this.metadataTableDao = metadataTableDao;
+ this.endTime = endTime;
+ this.processNewPartitionsAction = processNewPartitionsAction;
+ }
+
+ /**
+ * Resume from previously drained pipeline. Restart StreamPartitions by reading the last metadata
+ * write and output PartitionRecord based on that. Process NewPartitions normally.
+ *
+ * <ol>
+ * <li>Clear out missing partitions metadata in the metadata table
+ * <li>Read all StreamPartitions and output them
+ * <li>Read all NewPartitions and output them normally
+ * </ol>
+ *
+ * @param receiver output receiver to output
+ * @throws InvalidProtocolBufferException when failing to read from metadata table.
+ */
+ public void run(OutputReceiver<PartitionRecord> receiver) throws InvalidProtocolBufferException {
+ // Erase any existing missing partitions.
+ metadataTableDao.writeDetectNewPartitionMissingPartitions(new HashMap<>());
+
+ List<PartitionRecord> partitions = metadataTableDao.readAllStreamPartitions();
+
+ for (PartitionRecord partitionRecord : partitions) {
+ if (partitionRecord.getUuid().isEmpty()) {
+ partitionRecord.setUuid(UniqueIdGenerator.getNextId());
+ }
+ if (endTime != null) {
+ partitionRecord.setEndTime(endTime);
+ }
+ LOG.info("DNP: Outputting existing partition: {}", partitionRecord);
+ metrics.incListPartitionsCount();
+ receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
+ }
+
+ List<NewPartition> newPartitions = metadataTableDao.readNewPartitionsIncludingDeleted();
+
+ for (NewPartition newPartition : newPartitions) {
+ processNewPartitionsAction.processNewPartition(newPartition, receiver);
+ }
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java
index 29d8d72d9f6..69c6efba1de 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java
@@ -110,12 +110,6 @@ class BigtableChangeStreamAccessor {
EnhancedBigtableStubSettings.defaultGrpcTransportProviderBuilder()
.setAttemptDirectPath(false) // Disable DirectPath
.setChannelPoolSettings( // Autoscale Channel Size
- // We use blocking rpc calls so we expect a max of `workerHarnessThreads`
- // concurrent rpcs per-worker. This defaults to 500 and we want to accommodate
- // higher concurrency for jobs that are overridden to use more threads. There
- // is a limit of 100 concurrent streams per channel so this should resize to
- // support a large number of requests. Testing has shown that this
- // significantly increases throughput compared to the default channel pool.
ChannelPoolSettings.builder()
// Make sure that there are at least 2 channels regardless of RPCs
.setMinChannelCount(2)
@@ -191,9 +185,6 @@ class BigtableChangeStreamAccessor {
.readChangeStreamSettings()
.setRetrySettings(
readChangeStreamRetrySettings
- // Set timeouts to 60s - dataflow should checkpoint before then, but it is not
- // guaranteed to happen after a specific duration. We still want a conservative
- // timeout so that it can't hang.
.setInitialRpcTimeout(Duration.ofSeconds(60))
.setTotalTimeout(Duration.ofSeconds(60))
.setMaxRpcTimeout(Duration.ofSeconds(60))
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java
index f47d0f6370e..1c24e09db7d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java
@@ -89,7 +89,6 @@ public class ChangeStreamDao {
query.continuationTokens(Collections.singletonList(currentToken));
} else if (startTime != null) {
query.startTime(toThreetenInstant(startTime));
- // Check if partition has Continuation Tokens
} else if (changeStreamContinuationTokenList != null) {
query.continuationTokens(changeStreamContinuationTokenList);
} else {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java
index b826789fef4..acecc22b891 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java
@@ -92,11 +92,10 @@ public class DaoFactory implements Serializable {
if (changeStreamDao == null) {
checkArgumentNotNull(changeStreamConfig.getProjectId());
checkArgumentNotNull(changeStreamConfig.getInstanceId());
- String tableId = this.tableId;
checkArgumentNotNull(changeStreamConfig.getAppProfileId());
BigtableDataClient dataClient =
BigtableChangeStreamAccessor.getOrCreate(changeStreamConfig).getDataClient();
- changeStreamDao = new ChangeStreamDao(dataClient, tableId);
+ changeStreamDao = new ChangeStreamDao(dataClient, this.tableId);
}
return changeStreamDao;
}
@@ -122,7 +121,7 @@ public class DaoFactory implements Serializable {
if (metadataTableAdminDao == null) {
checkArgumentNotNull(metadataTableConfig.getProjectId());
checkArgumentNotNull(metadataTableConfig.getInstanceId());
- String tableId = checkArgumentNotNull(this.metadataTableId);
+ checkArgumentNotNull(this.metadataTableId);
checkArgumentNotNull(metadataTableConfig.getAppProfileId());
BigtableTableAdminClient tableAdminClient =
BigtableChangeStreamAccessor.getOrCreate(metadataTableConfig).getTableAdminClient();
@@ -130,7 +129,7 @@ public class DaoFactory implements Serializable {
BigtableChangeStreamAccessor.getOrCreate(metadataTableConfig).getInstanceAdminClient();
metadataTableAdminDao =
new MetadataTableAdminDao(
- tableAdminClient, instanceAdminClient, changeStreamName, tableId);
+ tableAdminClient, instanceAdminClient, changeStreamName, this.metadataTableId);
}
return metadataTableAdminDao;
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java
index d2f3872d329..5194ca49d62 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java
@@ -34,10 +34,10 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Immutabl
/**
* Data access object for creating and dropping the metadata table.
*
- * <p>The metadata table will be used to keep the state of the entire Dataflow job as well as
+ * <p>The metadata table will be used to keep the state of the entire Beam pipeline as well as
* splitting and merging partitions.
*
- * <p>Each Dataflow job will create its own metadata table.
+ * <p>Each Beam pipeline will create its own metadata table.
*/
@Internal
public class MetadataTableAdminDao {
@@ -50,6 +50,7 @@ public class MetadataTableAdminDao {
public static final String CF_LOCK = "lock";
public static final String CF_MISSING_PARTITIONS = "missing_partitions";
public static final String CF_VERSION = "version";
+ public static final String CF_SHOULD_DELETE = "should_delete";
public static final String QUALIFIER_DEFAULT = "latest";
public static final ImmutableList<String> COLUMN_FAMILIES =
ImmutableList.of(
@@ -60,7 +61,8 @@ public class MetadataTableAdminDao {
CF_CONTINUATION_TOKEN,
CF_LOCK,
CF_MISSING_PARTITIONS,
- CF_VERSION);
+ CF_VERSION,
+ CF_SHOULD_DELETE);
public static final ByteString NEW_PARTITION_PREFIX = ByteString.copyFromUtf8("NewPartition#");
public static final ByteString STREAM_PARTITION_PREFIX =
ByteString.copyFromUtf8("StreamPartition#");
@@ -170,4 +172,32 @@ public class MetadataTableAdminDao {
tableAdminClient.createTable(createTableRequest);
return true;
}
+
+ /**
+ * Delete all the metadata rows starting with the change stream name prefix, except for detect new
+ * partition row because it signals the existence of a pipeline with the change stream name.
+ */
+ public void cleanUpPrefix() {
+ // Clean up all rows except for DNP row.
+ tableAdminClient.dropRowRange(tableId, getFullNewPartitionPrefix());
+ tableAdminClient.dropRowRange(tableId, getFullStreamPartitionPrefix());
+ }
+
+ /**
+ * Return new partition row key prefix concatenated with change stream name.
+ *
+ * @return new partition row key prefix concatenated with change stream name.
+ */
+ private ByteString getFullNewPartitionPrefix() {
+ return changeStreamNamePrefix.concat(NEW_PARTITION_PREFIX);
+ }
+
+ /**
+ * Return stream partition row key prefix concatenated with change stream name.
+ *
+ * @return stream partition row key prefix concatenated with change stream name.
+ */
+ private ByteString getFullStreamPartitionPrefix() {
+ return changeStreamNamePrefix.concat(STREAM_PARTITION_PREFIX);
+ }
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java
index 30abd6d88a1..295184e1cc1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java
@@ -18,28 +18,50 @@
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao;
import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.microsecondToInstant;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao.NEW_PARTITION_PREFIX;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao.STREAM_PARTITION_PREFIX;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.isRowLocked;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseInitialContinuationTokens;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseLockUuid;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseTokenFromRow;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseWatermarkFromRow;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseWatermarkLastUpdatedFromRow;
import com.google.api.gax.rpc.ServerStream;
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation;
+import com.google.cloud.bigtable.data.v2.models.Filters;
import com.google.cloud.bigtable.data.v2.models.Filters.Filter;
import com.google.cloud.bigtable.data.v2.models.Mutation;
import com.google.cloud.bigtable.data.v2.models.Query;
import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowCell;
import com.google.cloud.bigtable.data.v2.models.RowMutation;
import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.beam.repackaged.core.org.apache.commons.lang3.SerializationException;
import org.apache.beam.repackaged.core.org.apache.commons.lang3.SerializationUtils;
import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.DetectNewPartitionsState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.StreamPartitionWithWatermark;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs;
+import org.joda.time.Duration;
import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,6 +76,11 @@ import org.slf4j.LoggerFactory;
public class MetadataTableDao {
private static final Logger LOG = LoggerFactory.getLogger(MetadataTableDao.class);
+ // New partition marked for deletion should be reevaluated after this delay because it might be
+ // output has failed.
+ private static final Duration DELETED_NEW_PARTITION_REEVALUATE_DELAY =
+ Duration.standardMinutes(1);
+
private final BigtableDataClient dataClient;
private final String tableId;
private final ByteString changeStreamNamePrefix;
@@ -150,40 +177,123 @@ public class MetadataTableDao {
}
/**
- * @return stream of all the new partitions resulting from splits and merges waiting to be
- * streamed.
+ * Read the low watermark of the pipeline from Detect New Partition row.
+ *
+ * @return DetectNewPartitions row from the metadata table.
+ */
+ public @Nullable DetectNewPartitionsState readDetectNewPartitionsState() {
+ Row row =
+ dataClient.readRow(
+ tableId,
+ getFullDetectNewPartition(),
+ FILTERS
+ .chain()
+ .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_WATERMARK))
+ .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
+ .filter(FILTERS.limit().cellsPerColumn(1)));
+ if (row == null) {
+ return null;
+ }
+ Instant watermark = parseWatermarkFromRow(row);
+ Instant timestamp = parseWatermarkLastUpdatedFromRow(row);
+ if (watermark == null || timestamp == null) {
+ return null;
+ }
+ return new DetectNewPartitionsState(watermark, timestamp);
+ }
+
+ /**
+ * @return all the new partitions resulting from splits and merges waiting to be streamed
+ * including ones marked for deletion.
*/
- public ServerStream<Row> readNewPartitions() {
- // It's important that we limit to the latest value per column because it's possible to write to
- // the same column multiple times. We don't want to read and send duplicate tokens to the
- // server.
+ public List<NewPartition> readNewPartitionsIncludingDeleted()
+ throws InvalidProtocolBufferException {
+ List<NewPartition> newPartitions = new ArrayList<>();
Query query =
Query.create(tableId)
.prefix(getFullNewPartitionPrefix())
- .filter(FILTERS.limit().cellsPerColumn(1));
- return dataClient.readRows(query);
+ .filter(Filters.FILTERS.limit().cellsPerColumn(1));
+ ServerStream<Row> rows = dataClient.readRows(query);
+ for (Row row : rows) {
+ ByteStringRange childPartition = convertNewPartitionRowKeyToPartition(row.getKey());
+ List<ChangeStreamContinuationToken> tokens = new ArrayList<>();
+ // Capture all the initial continuation tokens to request for the new partition.
+ for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_INITIAL_TOKEN)) {
+ ChangeStreamContinuationToken changeStreamContinuationToken =
+ ChangeStreamContinuationToken.fromByteString(cell.getValue());
+ tokens.add(changeStreamContinuationToken);
+ }
+ // There are no parents.
+ if (tokens.isEmpty()) {
+ continue;
+ }
+ // Capture all the low watermark and calculate the low watermark among all the parents.
+ List<Long> parentLowWatermark = new ArrayList<>();
+ for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS)) {
+ parentLowWatermark.add(Longs.fromByteArray(cell.getValue().toByteArray()));
+ }
+ Instant lowWatermark = Instant.ofEpochMilli(Collections.min(parentLowWatermark));
+ newPartitions.add(new NewPartition(childPartition, tokens, lowWatermark));
+ }
+ return newPartitions;
}
- /**
- * After a split or merge from a close stream, write the new partition's information to the
- * metadata table.
- *
- * @param newPartition the new partition
- * @param changeStreamContinuationToken the token that can be used to pick up from where the
- * parent left off
- * @param parentPartition the parent that stopped and split or merged
- * @param lowWatermark the low watermark of the parent stream
- */
- public void writeNewPartition(
- ByteStringRange newPartition,
- ChangeStreamContinuationToken changeStreamContinuationToken,
- ByteStringRange parentPartition,
- Instant lowWatermark) {
- writeNewPartition(
- newPartition,
- changeStreamContinuationToken.toByteString(),
- ByteStringRange.serializeToByteString(parentPartition),
- lowWatermark);
+ /** @return list the new partitions resulting from splits and merges waiting to be streamed. */
+ public List<NewPartition> readNewPartitions() throws InvalidProtocolBufferException {
+ List<NewPartition> newPartitions = new ArrayList<>();
+ Query query =
+ Query.create(tableId)
+ .prefix(getFullNewPartitionPrefix())
+ .filter(Filters.FILTERS.limit().cellsPerColumn(1));
+ ServerStream<Row> rows = dataClient.readRows(query);
+ for (Row row : rows) {
+ long lastUpdatedMicro = 0;
+ // We only process parent partition that are not marked for deletion.
+ Set<ByteStringRange> deletedParentPartitions = new HashSet<>();
+ for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_SHOULD_DELETE)) {
+ lastUpdatedMicro = Math.max(lastUpdatedMicro, cell.getTimestamp());
+ // Evaluate if the timestamp of the deletion marker is within the last 1 minute.
+ if (microsecondToInstant(cell.getTimestamp())
+ .plus(DELETED_NEW_PARTITION_REEVALUATE_DELAY)
+ .isAfterNow()) {
+ deletedParentPartitions.add(ByteStringRange.toByteStringRange(cell.getQualifier()));
+ }
+ }
+
+ ByteStringRange childPartition = convertNewPartitionRowKeyToPartition(row.getKey());
+ List<ChangeStreamContinuationToken> tokens = new ArrayList<>();
+ // Capture all the initial continuation tokens to request for the new partition.
+ for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_INITIAL_TOKEN)) {
+ // Only add parent partition that are not deleted.
+ if (!deletedParentPartitions.contains(
+ ByteStringRange.toByteStringRange(cell.getQualifier()))) {
+ ChangeStreamContinuationToken changeStreamContinuationToken =
+ ChangeStreamContinuationToken.fromByteString(cell.getValue());
+ tokens.add(changeStreamContinuationToken);
+ lastUpdatedMicro = Math.max(lastUpdatedMicro, cell.getTimestamp());
+ }
+ }
+ // All the parent partitions were skipped. We can skip this row.
+ if (tokens.isEmpty()) {
+ continue;
+ }
+
+ // Capture all the low watermark and calculate the low watermark among all the parents.
+ List<Long> parentLowWatermark = new ArrayList<>();
+ for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS)) {
+ // Only add parent partition that are not deleted.
+ if (!deletedParentPartitions.contains(
+ ByteStringRange.toByteStringRange(cell.getQualifier()))) {
+ parentLowWatermark.add(Longs.fromByteArray(cell.getValue().toByteArray()));
+ lastUpdatedMicro = Math.max(lastUpdatedMicro, cell.getTimestamp());
+ }
+ }
+ Instant lowWatermark = Instant.ofEpochMilli(Collections.min(parentLowWatermark));
+ newPartitions.add(
+ new NewPartition(
+ childPartition, tokens, lowWatermark, microsecondToInstant(lastUpdatedMicro)));
+ }
+ return newPartitions;
}
/**
@@ -191,45 +301,193 @@ public class MetadataTableDao {
* metadata table.
*
* @param newPartition the new partition
- * @param newPartitionContinuationToken continuation token for the new partition
- * @param parentPartition the parent that stopped
- * @param lowWatermark low watermark of the parent
*/
- private void writeNewPartition(
- ByteStringRange newPartition,
- ByteString newPartitionContinuationToken,
- ByteString parentPartition,
- Instant lowWatermark) {
- LOG.debug("Insert new partition");
- ByteString rowKey = convertPartitionToNewPartitionRowKey(newPartition);
+ public void writeNewPartition(NewPartition newPartition) {
+ ByteString rowKey = convertPartitionToNewPartitionRowKey(newPartition.getPartition());
+ ByteStringRange parentPartition =
+ newPartition.getChangeStreamContinuationTokens().get(0).getPartition();
RowMutation rowMutation =
RowMutation.create(tableId, rowKey)
- .setCell(MetadataTableAdminDao.CF_INITIAL_TOKEN, newPartitionContinuationToken, 1)
- .setCell(MetadataTableAdminDao.CF_PARENT_PARTITIONS, parentPartition, 1)
+ .setCell(
+ MetadataTableAdminDao.CF_INITIAL_TOKEN,
+ ByteStringRange.serializeToByteString(parentPartition),
+ newPartition.getChangeStreamContinuationTokens().get(0).toByteString())
.setCell(
MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS,
- parentPartition,
- lowWatermark.getMillis());
+ ByteStringRange.serializeToByteString(parentPartition),
+ newPartition.getLowWatermark().getMillis())
+ .deleteCells(
+ MetadataTableAdminDao.CF_SHOULD_DELETE,
+ ByteStringRange.serializeToByteString(parentPartition));
dataClient.mutateRow(rowMutation);
}
/**
- * @return stream of partitions currently being streamed by the beam job that have set a
- * watermark.
+ * This is the 1st step of 2 phase delete. Mark each parent partition in NewPartition for
+ * deletion. This avoids double reading the new partitions row for a while to allow RCSP to clean
+ * up partitions that led to it.
+ *
+ * <p>The reason behind 2 phase delete of NewPartition is to provide the invariant that the
+ * metadata table always has a copy of the ChangeStreamContinuationToken for every partition. The
+ * alternatives are either delete NewPartition <i>before</i> outputting PartitionRecord to RCSP or
+ * delete NewPartition <i>after</i> outputting PartitionRecord to RCSP.
+ *
+ * <p>The former has the problem that if the output failed and the NewPartition row has been
+ * deleted, the token is now lost.
+ *
+ * <p>The latter has a more complex problem of race between cleaning up NewPartition and RCSP
+ * writing StreamPartition. If clean up happens before StreamPartition is written, then at that
+ * moment the token is missing. While under normal operations, it should recover because RCSP
+ * writes StreamPartition eventually. We want to provide the invariant that there's always a copy
+ * of a token for every partition in the metadata table.
+ *
+ * @param newPartition mark for deletion.
*/
- public ServerStream<Row> readFromMdTableStreamPartitionsWithWatermark() {
- // We limit to the latest value per column.
+ public void markNewPartitionForDeletion(NewPartition newPartition) {
+ ByteString rowKey = convertPartitionToNewPartitionRowKey(newPartition.getPartition());
+ RowMutation rowMutation = RowMutation.create(tableId, rowKey);
+ for (ChangeStreamContinuationToken token : newPartition.getChangeStreamContinuationTokens()) {
+ rowMutation.setCell(
+ MetadataTableAdminDao.CF_SHOULD_DELETE,
+ ByteStringRange.serializeToByteString(token.getPartition()),
+ 1);
+ }
+ dataClient.mutateRow(rowMutation);
+ }
+
+ /**
+ * This is the 2nd step of 2 phase delete. Delete the newPartition cells. This should take place
+ * after the tokens have been written to StreamPartition.
+ *
+ * <p>It's possible to try to delete multiple parent partition cells but only a subset are marked
+ * for deletion. Only the cells marked for deletion are deleted.
+ *
+ * @param newPartition row that represents the new partition.
+ * @return true if successfully deleted entire newPartition.
+ */
+ public boolean deleteNewPartition(NewPartition newPartition) {
+ ByteString rowKey = convertPartitionToNewPartitionRowKey(newPartition.getPartition());
+
+ boolean success = true;
+
+ for (ChangeStreamContinuationToken token : newPartition.getChangeStreamContinuationTokens()) {
+ Filter shouldDelete =
+ FILTERS
+ .chain()
+ .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_SHOULD_DELETE))
+ .filter(
+ FILTERS
+ .qualifier()
+ .exactMatch(ByteStringRange.serializeToByteString(token.getPartition())))
+ .filter(FILTERS.value().exactMatch(ByteString.copyFrom(Longs.toByteArray(1))));
+ Mutation mutation =
+ Mutation.create()
+ .deleteCells(
+ MetadataTableAdminDao.CF_INITIAL_TOKEN,
+ ByteStringRange.serializeToByteString(token.getPartition()))
+ .deleteCells(
+ MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS,
+ ByteStringRange.serializeToByteString(token.getPartition()))
+ .deleteCells(
+ MetadataTableAdminDao.CF_SHOULD_DELETE,
+ ByteStringRange.serializeToByteString(token.getPartition()));
+
+ ConditionalRowMutation conditionalRowMutation =
+ ConditionalRowMutation.create(tableId, rowKey).condition(shouldDelete).then(mutation);
+ success = dataClient.checkAndMutateRow(conditionalRowMutation) && success;
+ }
+
+ return success;
+ }
+
+ /**
+ * Return list of locked StreamPartition and their watermarks.
+ *
+ * @return list of partitions currently being streamed that have a watermark.
+ */
+ public List<StreamPartitionWithWatermark> readStreamPartitionsWithWatermark()
+ throws InvalidProtocolBufferException {
+ LOG.debug(
+ "Reading stream partitions from metadata table: "
+ + getFullStreamPartitionPrefix().toStringUtf8());
+ Filter filterForWatermark =
+ FILTERS
+ .chain()
+ .filter(Filters.FILTERS.limit().cellsPerColumn(1))
+ .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_WATERMARK))
+ .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT));
+ Filter filterForLock =
+ FILTERS
+ .chain()
+ .filter(Filters.FILTERS.limit().cellsPerColumn(1))
+ .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_LOCK))
+ .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT));
Query query =
Query.create(tableId)
.prefix(getFullStreamPartitionPrefix())
- .filter(
- FILTERS
- .chain()
- .filter(FILTERS.limit().cellsPerColumn(1))
- .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_WATERMARK))
- .filter(
- FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT)));
- return dataClient.readRows(query);
+ .filter(FILTERS.interleave().filter(filterForWatermark).filter(filterForLock));
+ ServerStream<Row> rows = dataClient.readRows(query);
+ List<StreamPartitionWithWatermark> partitions = new ArrayList<>();
+ for (Row row : rows) {
+ if (!isRowLocked(row)) {
+ continue;
+ }
+ Instant watermark = MetadataTableEncoder.parseWatermarkFromRow(row);
+ if (watermark == null) {
+ continue;
+ }
+ ByteStringRange partition = convertStreamPartitionRowKeyToPartition(row.getKey());
+ partitions.add(new StreamPartitionWithWatermark(partition, watermark));
+ }
+ return partitions;
+ }
+
+ /**
+ * Read all the StreamPartition and output PartitionRecord to stream them.
+ *
+ * @return list of PartitionRecord of all StreamPartitions in the metadata table.
+ */
+ public List<PartitionRecord> readAllStreamPartitions() throws InvalidProtocolBufferException {
+ Query query = Query.create(tableId).prefix(getFullStreamPartitionPrefix());
+ ServerStream<Row> rows = dataClient.readRows(query);
+ List<PartitionRecord> partitions = new ArrayList<>();
+ for (Row row : rows) {
+ Instant watermark = parseWatermarkFromRow(row);
+ String uuid = parseLockUuid(row);
+ ByteStringRange partition = convertStreamPartitionRowKeyToPartition(row.getKey());
+ String currentToken = parseTokenFromRow(row);
+ List<ChangeStreamContinuationToken> initialTokens = parseInitialContinuationTokens(row);
+ if (watermark == null) {
+ // This is unexpected. Watermark should never be null. We can't just leave this around
+ // either, it will cause conflicts. We need to delete it.
+ if (uuid != null) {
+ releaseStreamPartitionLockForDeletion(partition, uuid);
+ }
+ deleteStreamPartitionRow(partition);
+ LOG.error("Cleaning up corrupted StreamPartition {}", formatByteStringRange(partition));
+ continue;
+ }
+ PartitionRecord partitionRecord;
+ if (currentToken != null) {
+ ChangeStreamContinuationToken token =
+ ChangeStreamContinuationToken.create(partition, currentToken);
+ partitionRecord =
+ new PartitionRecord(
+ partition, Collections.singletonList(token), watermark, Collections.emptyList());
+ } else if (!initialTokens.isEmpty()) {
+ partitionRecord =
+ new PartitionRecord(partition, initialTokens, watermark, Collections.emptyList());
+ } else {
+ partitionRecord =
+ new PartitionRecord(partition, watermark, watermark, Collections.emptyList());
+ }
+
+ if (uuid != null) {
+ partitionRecord.setUuid(uuid);
+ }
+ partitions.add(partitionRecord);
+ }
+ return partitions;
}
/**
@@ -256,6 +514,15 @@ public class MetadataTableDao {
dataClient.mutateRow(rowMutation);
}
+ /**
+ * Update the watermark cell for Detect New Partition step.
+ *
+ * @param watermark watermark value to set for the cell
+ */
+ public void updateDetectNewPartitionWatermark(Instant watermark) {
+ writeToMdTableWatermarkHelper(getFullDetectNewPartition(), watermark, null);
+ }
+
/**
* Update the metadata for the row key represented by the partition.
*
@@ -272,41 +539,103 @@ public class MetadataTableDao {
}
/**
- * Delete the row key represented by the partition. This represents that the partition will no
- * longer be streamed.
+ * This is the 1st step of 2 phase delete of StreamPartition. Unset the uuid of StreamPartition if
+ * the uuid matches and set the deletion bit. This prepares the StreamPartition to be deleted.
*
- * @param partition forms the row key of the row to delete
+ * <p>The reason to have 2 phase delete is
+ *
+ * <ul>
+ * <li>If we delete StreamPartition <b>before</b> we write NewPartitions, then we can't satisfy
+ * the invariant that there always exists a continuation token for every row key; no matter
+ * how temporary.
+ * <li>If we delete StreamPartition <b>after</b> we write NewPartitions, we allow a more complex
+ * and rare race problem. After we write the NewPartition but before we delete
+ * StreamPartition the following series of event take place. DNP processes the
+ * NewPartitions. The NewPartitions start streaming and then immediately merge back to the
+ * same partition as the partition we're about to Delete. The new partition tries to lock
+ * the StreamPartition row. Locking fails because the lock is held by the RCSP that we're
+ * deleting.
+ * <p>For example: the RCSP with id <code>1234</code> is working on partition AC. Metadata
+ * table has the row <code>
+ * StreamPartition#AC</code> with <code>uuid=1234</code>. Partition AC splits into AB and
+ * BC. We write <code>NewPartition#AB</code> and <code>
+ * NewPartition#BC</code>. But we haven't deleted <code>StreamPartition#AC</code> yet.
+ * Before we clean up <code>StreamPartition#AC</code>, DNP processes AB and BC. AB and BC
+ * start streaming. AB and BC immediately merge back to AC. DNP processes the <code>
+ * NewPartition#AC</code> and outputs AC. A new RCSP with id 5678 tries to lock <code>
+ * StreamPartition#AC</code> to start processing AC. This fails because RCSP 1234 still
+ * holds the lock. Once the lock fails RCSP 5678 stops. RCSP 1234 eventually cleans up
+ * <code>StreamPartition#AC</code> and no RCSP is streaming AC anymore.
+ * </ul>
+ *
+ * The solution is a 2 phase delete
+ *
+ * <ol>
+ * <li>Release the lock and mark the StreamPartition for deletion
+ * <li>Write NewPartition rows
+ * <li>Delete StreamPartition if and only if deletion bit is set
+ * </ol>
+ *
+ * We added the deletion bit to prevent accidentally deleting a new StreamPartition as the result
+ * of the split and merge race described above. If the above scenario happens, the new partition
+ * will be able to lock the StreamPartition row. But we don't want to delete the StreamPartition,
+ * so we check if it is still marked for deletion. Obviously, locking the row unsets the deletion
+ * bit, so it can't be deleted.
+ *
+ * @param partition release the lock for this partition
+ * @param uuid match the uuid
+ * @return true if releasing the lock was successful.
*/
- public void deleteStreamPartitionRow(ByteStringRange partition) {
+ public boolean releaseStreamPartitionLockForDeletion(ByteStringRange partition, String uuid) {
ByteString rowKey = convertPartitionToStreamPartitionRowKey(partition);
- RowMutation rowMutation = RowMutation.create(tableId, rowKey).deleteRow();
- dataClient.mutateRow(rowMutation);
+ Filter lockCellFilter =
+ FILTERS
+ .chain()
+ .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_LOCK))
+ .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
+ .filter(FILTERS.value().exactMatch(uuid));
+ Mutation deleteCell =
+ Mutation.create()
+ .deleteCells(MetadataTableAdminDao.CF_LOCK, MetadataTableAdminDao.QUALIFIER_DEFAULT)
+ .setCell(
+ MetadataTableAdminDao.CF_SHOULD_DELETE, MetadataTableAdminDao.QUALIFIER_DEFAULT, 1);
+ ConditionalRowMutation rowMutation =
+ ConditionalRowMutation.create(tableId, rowKey).condition(lockCellFilter).then(deleteCell);
+ return dataClient.checkAndMutateRow(rowMutation);
}
/**
- * Delete the row.
+ * This is the 2nd step of 2 phase delete of StreamPartition. Delete the row key represented by
+ * the partition. This represents that the partition will no longer be streamed. Only delete if
+ * shouldDelete bit is set.
*
- * @param rowKey row key of the row to delete
+ * @param partition forms the row key of the row to delete
*/
- public void deleteRowKey(ByteString rowKey) {
- RowMutation rowMutation = RowMutation.create(tableId, rowKey).deleteRow();
- dataClient.mutateRow(rowMutation);
+ public boolean deleteStreamPartitionRow(ByteStringRange partition) {
+ LOG.debug("Delete metadata row");
+ ByteString rowKey = convertPartitionToStreamPartitionRowKey(partition);
+ Filter shouldDeleteFilter =
+ FILTERS
+ .chain()
+ .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_SHOULD_DELETE))
+ .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
+ .filter(FILTERS.value().exactMatch(ByteString.copyFrom(Longs.toByteArray(1))));
+ Mutation deleteRow = Mutation.create().deleteRow();
+ ConditionalRowMutation rowMutation =
+ ConditionalRowMutation.create(tableId, rowKey)
+ .condition(shouldDeleteFilter)
+ .then(deleteRow);
+ return dataClient.checkAndMutateRow(rowMutation);
}
/**
- * Lock the partition in the metadata table for the DoFn streaming it. Only one DoFn is allowed to
- * stream a specific partition at any time. Each DoFn has an uuid and will try to lock the
- * partition at the very start of the stream. If another DoFn has already locked the partition
- * (i.e. the uuid in the cell for the partition belongs to the DoFn), any future DoFn trying to
- * lock the same partition will and terminate.
+ * Return true if the uuid holds the lock of the partition.
*
- * @param partition form the row key in the metadata table to lock
- * @param uuid id of the DoFn
+ * @param partition partition to check if lock is held
+ * @param uuid to check if it holds the lock
* @return true if uuid holds the lock, otherwise false.
*/
- public boolean lockPartition(ByteStringRange partition, String uuid) {
- LOG.debug("Locking partition before processing stream");
-
+ public boolean doHoldLock(ByteStringRange partition, String uuid) {
ByteString rowKey = convertPartitionToStreamPartitionRowKey(partition);
Filter lockCellFilter =
FILTERS
@@ -315,18 +644,53 @@ public class MetadataTableDao {
.filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
.filter(FILTERS.limit().cellsPerRow(1));
Row row = dataClient.readRow(tableId, rowKey, lockCellFilter);
-
- // If the query returns non-null row, that means the lock is being held. Check if the owner is
- // same as uuid.
if (row != null) {
return row.getCells().get(0).getValue().toStringUtf8().equals(uuid);
}
+ return false;
+ }
- // We cannot check whether a cell is empty, We can check if the cell matches any value. If it
- // does not, we perform the mutation to set the cell.
+ /**
+ * Lock the partition in the metadata table for the DoFn streaming it. Only one DoFn is allowed to
+ * stream a specific partition at any time. Each DoFn has an uuid and will try to lock the
+ * partition at the very start of the stream. If another DoFn has already locked the partition
+ * (i.e. the uuid in the cell for the partition belongs to the DoFn), any future DoFn trying to
+ * lock the same partition will fail. Also unset the deletion bit.
+ *
+ * @param partitionRecord partition to lock
+ * @return true if uuid holds or acquired the lock, otherwise false.
+ */
+ public boolean lockAndRecordPartition(PartitionRecord partitionRecord) {
+ if (doHoldLock(partitionRecord.getPartition(), partitionRecord.getUuid())) {
+ return true;
+ }
+
+ // Record all the initial metadata.
Mutation mutation =
Mutation.create()
- .setCell(MetadataTableAdminDao.CF_LOCK, MetadataTableAdminDao.QUALIFIER_DEFAULT, uuid);
+ .setCell(
+ MetadataTableAdminDao.CF_LOCK,
+ MetadataTableAdminDao.QUALIFIER_DEFAULT,
+ partitionRecord.getUuid())
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK,
+ MetadataTableAdminDao.QUALIFIER_DEFAULT,
+ partitionRecord.getParentLowWatermark().getMillis())
+ .deleteCells(
+ MetadataTableAdminDao.CF_SHOULD_DELETE, MetadataTableAdminDao.QUALIFIER_DEFAULT);
+ List<ChangeStreamContinuationToken> tokens =
+ partitionRecord.getChangeStreamContinuationTokens();
+ if (tokens != null) {
+ for (ChangeStreamContinuationToken token : tokens) {
+ mutation.setCell(
+ MetadataTableAdminDao.CF_INITIAL_TOKEN,
+ ByteStringRange.serializeToByteString(token.getPartition()),
+ token.toByteString());
+ }
+ }
+
+ // We cannot check whether a cell is empty, We can check if the cell matches any value. If it
+ // does not, we perform the mutation to set the cell.
Filter matchAnyString =
FILTERS
.chain()
@@ -334,7 +698,8 @@ public class MetadataTableDao {
.filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
.filter(FILTERS.value().regex("\\C*"));
ConditionalRowMutation rowMutation =
- ConditionalRowMutation.create(tableId, rowKey)
+ ConditionalRowMutation.create(
+ tableId, convertPartitionToStreamPartitionRowKey(partitionRecord.getPartition()))
.condition(matchAnyString)
.otherwise(mutation);
return !dataClient.checkAndMutateRow(rowMutation);
@@ -360,8 +725,8 @@ public class MetadataTableDao {
*
* @return deserialized missing partitions and duration.
*/
- public HashMap<ByteStringRange, Long> readDetectNewPartitionMissingPartitions() {
- @Nonnull HashMap<ByteStringRange, Long> missingPartitions = new HashMap<>();
+ public HashMap<ByteStringRange, Instant> readDetectNewPartitionMissingPartitions() {
+ @Nonnull HashMap<ByteStringRange, Instant> missingPartitions = new HashMap<>();
Filter missingPartitionsFilter =
FILTERS
.chain()
@@ -397,7 +762,7 @@ public class MetadataTableDao {
* @param missingPartitionDurations missing partitions and duration.
*/
public void writeDetectNewPartitionMissingPartitions(
- HashMap<ByteStringRange, Long> missingPartitionDurations) {
+ HashMap<ByteStringRange, Instant> missingPartitionDurations) {
byte[] serializedMissingPartition = SerializationUtils.serialize(missingPartitionDurations);
RowMutation rowMutation =
RowMutation.create(tableId, getFullDetectNewPartition())
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/DetectNewPartitionsDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/DetectNewPartitionsDoFn.java
index 5433ee86fc4..1ec1abb323a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/DetectNewPartitionsDoFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/DetectNewPartitionsDoFn.java
@@ -24,10 +24,14 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ActionFactory;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.DetectNewPartitionsAction;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.GenerateInitialPartitionsAction;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ProcessNewPartitionsAction;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ResumeFromPreviousPipelineAction;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.DetectNewPartitionsTracker;
import org.apache.beam.sdk.io.range.OffsetRange;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement;
@@ -41,7 +45,7 @@ import org.joda.time.Instant;
@SuppressWarnings("initialization.fields.uninitialized")
@Internal
@UnboundedPerElement
-public class DetectNewPartitionsDoFn extends DoFn<Instant, PartitionRecord> {
+public class DetectNewPartitionsDoFn extends DoFn<InitialPipelineState, PartitionRecord> {
private static final long serialVersionUID = 8052524268978107367L;
@Nullable private final Instant endTime;
@@ -62,8 +66,9 @@ public class DetectNewPartitionsDoFn extends DoFn<Instant, PartitionRecord> {
}
@GetInitialWatermarkEstimatorState
- public Instant getInitialWatermarkEstimatorState(@Element Instant startTime) {
- return startTime;
+ public Instant getInitialWatermarkEstimatorState(
+ @Element InitialPipelineState initialPipelineState) {
+ return initialPipelineState.getStartTime();
}
@NewWatermarkEstimator
@@ -79,7 +84,7 @@ public class DetectNewPartitionsDoFn extends DoFn<Instant, PartitionRecord> {
@NewTracker
public OffsetRangeTracker restrictionTracker(@Restriction OffsetRange restriction) {
- return new OffsetRangeTracker(restriction);
+ return new DetectNewPartitionsTracker(restriction.getFrom());
}
// We never want to scale based on this DoFn, so we return a constant backlog estimate of zero.
@@ -92,22 +97,31 @@ public class DetectNewPartitionsDoFn extends DoFn<Instant, PartitionRecord> {
public void setup() throws IOException {
final MetadataTableDao metadataTableDao = daoFactory.getMetadataTableDao();
final ChangeStreamDao changeStreamDao = daoFactory.getChangeStreamDao();
+ ProcessNewPartitionsAction processNewPartitionsAction =
+ actionFactory.processNewPartitionsAction(metrics, metadataTableDao, endTime);
GenerateInitialPartitionsAction generateInitialPartitionsAction =
actionFactory.generateInitialPartitionsAction(metrics, changeStreamDao, endTime);
+ ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction =
+ actionFactory.resumeFromPreviousPipelineAction(
+ metrics, metadataTableDao, endTime, processNewPartitionsAction);
detectNewPartitionsAction =
actionFactory.detectNewPartitionsAction(
- metrics, metadataTableDao, endTime, generateInitialPartitionsAction);
+ metrics,
+ metadataTableDao,
+ endTime,
+ generateInitialPartitionsAction,
+ resumeFromPreviousPipelineAction,
+ processNewPartitionsAction);
}
@ProcessElement
public ProcessContinuation processElement(
- @Element Instant startTime,
+ @Element InitialPipelineState initialPipelineState,
RestrictionTracker<OffsetRange, Long> tracker,
OutputReceiver<PartitionRecord> receiver,
- ManualWatermarkEstimator<Instant> watermarkEstimator,
- BundleFinalizer bundleFinalizer)
+ ManualWatermarkEstimator<Instant> watermarkEstimator)
throws Exception {
return detectNewPartitionsAction.run(
- tracker, receiver, watermarkEstimator, bundleFinalizer, startTime);
+ tracker, receiver, watermarkEstimator, initialPipelineState);
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java
index 51ef62bcd4e..07826ee1db7 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java
@@ -20,7 +20,10 @@ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn;
import java.io.IOException;
import java.io.Serializable;
import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO.ExistingPipelineOptions;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.DetectNewPartitionsState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
import org.apache.beam.sdk.transforms.DoFn;
import org.joda.time.Instant;
import org.slf4j.Logger;
@@ -31,23 +34,28 @@ import org.slf4j.LoggerFactory;
* pipeline.
*/
@Internal
-public class InitializeDoFn extends DoFn<byte[], Instant> implements Serializable {
+public class InitializeDoFn extends DoFn<byte[], InitialPipelineState> implements Serializable {
private static final long serialVersionUID = 1868189906451252363L;
private static final Logger LOG = LoggerFactory.getLogger(InitializeDoFn.class);
private final DaoFactory daoFactory;
private final String metadataTableAppProfileId;
private Instant startTime;
+ private final ExistingPipelineOptions existingPipelineOptions;
public InitializeDoFn(
- DaoFactory daoFactory, String metadataTableAppProfileId, Instant startTime) {
+ DaoFactory daoFactory,
+ String metadataTableAppProfileId,
+ Instant startTime,
+ ExistingPipelineOptions existingPipelineOptions) {
this.daoFactory = daoFactory;
this.metadataTableAppProfileId = metadataTableAppProfileId;
this.startTime = startTime;
+ this.existingPipelineOptions = existingPipelineOptions;
}
@ProcessElement
- public void processElement(OutputReceiver<Instant> receiver) throws IOException {
+ public void processElement(OutputReceiver<InitialPipelineState> receiver) throws IOException {
LOG.info(daoFactory.getStreamTableDebugString());
LOG.info(daoFactory.getMetadataTableDebugString());
LOG.info("ChangeStreamName: " + daoFactory.getChangeStreamName());
@@ -69,8 +77,64 @@ public class InitializeDoFn extends DoFn<byte[], Instant> implements Serializabl
"Reusing existing metadata table: " + daoFactory.getMetadataTableAdminDao().getTableId());
}
- daoFactory.getMetadataTableDao().writeDetectNewPartitionVersion();
+ boolean resume = false;
+ DetectNewPartitionsState detectNewPartitionsState =
+ daoFactory.getMetadataTableDao().readDetectNewPartitionsState();
- receiver.output(startTime);
+ switch (existingPipelineOptions) {
+ case NEW:
+ // clean up table
+ LOG.info(
+ "Cleaning up an old pipeline with the same change stream name to start a new pipeline with the same name.");
+ daoFactory.getMetadataTableAdminDao().cleanUpPrefix();
+ break;
+ case RESUME_OR_NEW:
+ // perform resumption.
+ if (detectNewPartitionsState != null) {
+ resume = true;
+ startTime = detectNewPartitionsState.getWatermark();
+ LOG.info("Resuming from previous pipeline with low watermark of {}", startTime);
+ } else {
+ LOG.info(
+ "Attempted to resume, but previous watermark does not exist, starting at {}",
+ startTime);
+ daoFactory.getMetadataTableAdminDao().cleanUpPrefix();
+ }
+ break;
+ case RESUME_OR_FAIL:
+ // perform resumption.
+ if (detectNewPartitionsState != null) {
+ resume = true;
+ startTime = detectNewPartitionsState.getWatermark();
+ LOG.info("Resuming from previous pipeline with low watermark of {}", startTime);
+ } else {
+ LOG.error("Previous pipeline with the same change stream name doesn't exist, stopping");
+ return;
+ }
+ break;
+ case FAIL_IF_EXISTS:
+ if (detectNewPartitionsState != null) {
+ LOG.error(
+ "A previous pipeline exists with the same change stream name and existingPipelineOption is set to FAIL_IF_EXISTS.");
+ return;
+ }
+ // We still want to clean up any existing prefixes in case there are lingering metadata that
+ // would interfere with the new run.
+ daoFactory.getMetadataTableAdminDao().cleanUpPrefix();
+ break;
+ case SKIP_CLEANUP:
+ if (detectNewPartitionsState != null) {
+ LOG.error(
+ "A previous pipeline exists with the same change stream name and existingPipelineOption is set to SKIP_CLEANUP. This option should only be used in tests.");
+ return;
+ }
+ break;
+ default:
+ LOG.error("Unexpected existingPipelineOptions option.");
+ // terminate pipeline
+ return;
+ }
+ daoFactory.getMetadataTableDao().writeDetectNewPartitionVersion();
+ receiver.output(new InitialPipelineState(startTime, resume));
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java
index cbd2895efe1..03cae41c5bd 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java
@@ -17,8 +17,13 @@
*/
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.microsecondToInstant;
+
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
import com.google.cloud.bigtable.data.v2.models.Row;
import com.google.cloud.bigtable.data.v2.models.RowCell;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.beam.sdk.annotations.Internal;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
@@ -44,13 +49,28 @@ public class MetadataTableEncoder {
return Instant.ofEpochMilli(Longs.fromByteArray(cells.get(0).getValue().toByteArray()));
}
+ /**
+ * Return the timestamp (the time it was updated) of the watermark cell.
+ *
+ * @param row row to extract the timestamp from
+ * @return the timestamp of the watermark cell.
+ */
+ public static @Nullable Instant parseWatermarkLastUpdatedFromRow(Row row) {
+ List<RowCell> cells =
+ row.getCells(MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT);
+ if (cells.size() == 0) {
+ return null;
+ }
+ return microsecondToInstant(cells.get(0).getTimestamp());
+ }
+
/**
* Read the continuation token cell of a row from ReadRows.
*
* @param row to extract the token from
* @return the token of the row
*/
- public static @Nullable String getTokenFromRow(Row row) {
+ public static @Nullable String parseTokenFromRow(Row row) {
List<RowCell> cells =
row.getCells(
MetadataTableAdminDao.CF_CONTINUATION_TOKEN, MetadataTableAdminDao.QUALIFIER_DEFAULT);
@@ -59,4 +79,48 @@ public class MetadataTableEncoder {
}
return cells.get(0).getValue().toStringUtf8();
}
+
+ /**
+ * Returns true if row is locked.
+ *
+ * @param row to evaluate if it's locked
+ * @return true if row is locked. Otherwise, false.
+ */
+ public static boolean isRowLocked(Row row) {
+ return row.getCells(MetadataTableAdminDao.CF_LOCK, MetadataTableAdminDao.QUALIFIER_DEFAULT)
+ .size()
+ != 0;
+ }
+
+ /**
+ * Returns the uuid from a row.
+ *
+ * @param row to extract the uuid
+ * @return the uuid of the row
+ */
+ public static @Nullable String parseLockUuid(Row row) {
+ List<RowCell> cells =
+ row.getCells(MetadataTableAdminDao.CF_LOCK, MetadataTableAdminDao.QUALIFIER_DEFAULT);
+ if (cells.size() == 0) {
+ return null;
+ }
+ return cells.get(0).getValue().toStringUtf8();
+ }
+
+ /**
+ * Return a list of initial token from a row.
+ *
+ * @param row to extract the initial tokens
+ * @return a list of tokens from the row
+ * @throws InvalidProtocolBufferException when failed to parse tokens.
+ */
+ public static List<ChangeStreamContinuationToken> parseInitialContinuationTokens(Row row)
+ throws InvalidProtocolBufferException {
+ List<ChangeStreamContinuationToken> tokens = new ArrayList<>();
+ List<RowCell> cells = row.getCells(MetadataTableAdminDao.CF_INITIAL_TOKEN);
+ for (RowCell rowCell : cells) {
+ tokens.add(ChangeStreamContinuationToken.fromByteString(rowCell.getValue()));
+ }
+ return tokens;
+ }
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/DetectNewPartitionsState.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/DetectNewPartitionsState.java
new file mode 100644
index 00000000000..2e43124d56c
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/DetectNewPartitionsState.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+import org.apache.beam.sdk.annotations.Internal;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.Instant;
+
+/**
+ * Metadata of the progress of {@link
+ * org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.DetectNewPartitionsDoFn} from the metadata
+ * table.
+ */
+@Internal
+public class DetectNewPartitionsState implements Serializable {
+ private static final long serialVersionUID = 6587699311321738060L;
+
+ private final Instant watermark;
+ private final Instant watermarkLastUpdated;
+
+ public DetectNewPartitionsState(Instant watermark, Instant watermarkLastUpdated) {
+ this.watermark = watermark;
+ this.watermarkLastUpdated = watermarkLastUpdated;
+ }
+
+ public Instant getWatermark() {
+ return watermark;
+ }
+
+ public Instant getWatermarkLastUpdated() {
+ return watermarkLastUpdated;
+ }
+
+ @Override
+ public boolean equals(@Nullable Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof DetectNewPartitionsState)) {
+ return false;
+ }
+ DetectNewPartitionsState that = (DetectNewPartitionsState) o;
+ return Objects.equals(getWatermark(), that.getWatermark())
+ && Objects.equals(getWatermarkLastUpdated(), that.getWatermarkLastUpdated());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getWatermark(), getWatermarkLastUpdated());
+ }
+
+ @Override
+ public String toString() {
+ return "DetectNewPartitions{"
+ + "watermark="
+ + watermark
+ + ", watermarkLastUpdated="
+ + watermarkLastUpdated
+ + '}';
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/InitialPipelineState.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/InitialPipelineState.java
new file mode 100644
index 00000000000..3f0da38a128
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/InitialPipelineState.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+import org.apache.beam.sdk.annotations.Internal;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.Instant;
+
+/**
+ * States to initialize a pipeline outputted by {@link
+ * org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.InitializeDoFn}.
+ */
+@Internal
+public class InitialPipelineState implements Serializable {
+ private static final long serialVersionUID = 7685843906645495071L;
+ private final Instant startTime;
+ private final boolean resume;
+
+ public InitialPipelineState(Instant startTime, boolean resume) {
+ this.startTime = startTime;
+ this.resume = resume;
+ }
+
+ public Instant getStartTime() {
+ return startTime;
+ }
+
+ public boolean isResume() {
+ return resume;
+ }
+
+ @Override
+ public boolean equals(@Nullable Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof InitialPipelineState)) {
+ return false;
+ }
+ InitialPipelineState that = (InitialPipelineState) o;
+ return isResume() == that.isResume() && Objects.equals(getStartTime(), that.getStartTime());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getStartTime(), isResume());
+ }
+
+ @Override
+ public String toString() {
+ return "InitialPipeline{" + "startTime=" + startTime + ", resume=" + resume + '}';
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/NewPartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/NewPartition.java
new file mode 100644
index 00000000000..08508172ad4
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/NewPartition.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.model;
+
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
+
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
+import org.joda.time.Instant;
+
+/**
+ * Represent new partition as a result of splits and merges.
+ *
+ * <p>Parent partitions stop streaming and record their change stream continuation tokens. The
+ * tokens are collected to continue the stream from the new partition.
+ */
+@Internal
+public class NewPartition implements Serializable {
+ private static final long serialVersionUID = -6530725494713332884L;
+ ByteStringRange partition;
+ List<ChangeStreamContinuationToken> changeStreamContinuationTokens;
+ Instant lowWatermark;
+ Instant lastUpdated;
+
+ public NewPartition(
+ ByteStringRange partition,
+ List<ChangeStreamContinuationToken> changeStreamContinuationTokens,
+ Instant lowWatermark,
+ Instant lastUpdated) {
+ this.partition = partition;
+ this.changeStreamContinuationTokens = changeStreamContinuationTokens;
+ this.lowWatermark = lowWatermark;
+ this.lastUpdated = lastUpdated;
+ }
+
+ public NewPartition(
+ ByteStringRange partition,
+ List<ChangeStreamContinuationToken> changeStreamContinuationTokens,
+ Instant lowWatermark) {
+ this(partition, changeStreamContinuationTokens, lowWatermark, Instant.EPOCH);
+ }
+
+ public ByteStringRange getPartition() {
+ return partition;
+ }
+
+ public List<ChangeStreamContinuationToken> getChangeStreamContinuationTokens() {
+ return changeStreamContinuationTokens;
+ }
+
+ public Instant getLowWatermark() {
+ return lowWatermark;
+ }
+
+ public List<ByteStringRange> getParentPartitions() {
+ return getChangeStreamContinuationTokens().stream()
+ .map(ChangeStreamContinuationToken::getPartition)
+ .collect(Collectors.toList());
+ }
+
+ public Instant getLastUpdated() {
+ return lastUpdated;
+ }
+
+ /**
+ * Return a new NewPartition that only contains one token that matches the parentPartition. If the
+ * parentPartition does not exist, return null.
+ *
+ * @param parentPartition to parent partition to find within the tokens.
+ * @return a NewPartition with one token with parentPartition. null otherwise.
+ */
+ public @Nullable NewPartition getSingleTokenNewPartition(ByteStringRange parentPartition) {
+ for (ChangeStreamContinuationToken parentToken : getChangeStreamContinuationTokens()) {
+ if (parentToken.getPartition().equals(parentPartition)) {
+ // The partition stays the same for the splices because that's the row key to find the cells
+ // to clean up.
+ return new NewPartition(
+ getPartition(),
+ Collections.singletonList(parentToken),
+ getLowWatermark(),
+ getLastUpdated());
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public boolean equals(@Nullable Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof NewPartition)) {
+ return false;
+ }
+ NewPartition that = (NewPartition) o;
+ return Objects.equals(getPartition(), that.getPartition())
+ && Objects.equals(
+ getChangeStreamContinuationTokens(), that.getChangeStreamContinuationTokens())
+ && Objects.equals(getLowWatermark(), that.getLowWatermark())
+ && Objects.equals(getLastUpdated(), that.getLastUpdated());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ getPartition(), getChangeStreamContinuationTokens(), getLowWatermark(), getLastUpdated());
+ }
+
+ @Override
+ public String toString() {
+ return "NewPartition{"
+ + "partition="
+ + formatByteStringRange(partition)
+ + ", changeStreamContinuationTokens="
+ + changeStreamContinuationTokens.stream()
+ .map(t -> formatByteStringRange(t.getPartition()) + " => {" + t.getToken() + "}")
+ .collect(Collectors.joining(", ", "{", "}"))
+ + ", lowWatermark="
+ + lowWatermark
+ + ", lastUpdated="
+ + lastUpdated
+ + '}';
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/PartitionRecord.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/PartitionRecord.java
index 82cd9b566a7..faf006b94e4 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/PartitionRecord.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/PartitionRecord.java
@@ -35,35 +35,45 @@ import org.joda.time.Instant;
*/
@Internal
public class PartitionRecord implements Serializable {
- private static final long serialVersionUID = -7613861834142734474L;
+ private static final long serialVersionUID = -4524061648930484599L;
- private ByteStringRange partition;
+ private final ByteStringRange partition;
@Nullable private Instant startTime;
@Nullable private List<ChangeStreamContinuationToken> changeStreamContinuationTokens;
@Nullable private Instant endTime;
private String uuid;
- private Instant parentLowWatermark;
+ private final Instant parentLowWatermark;
+ private final List<NewPartition> parentPartitions;
+
+ public PartitionRecord(
+ ByteStringRange partition,
+ List<ChangeStreamContinuationToken> changeStreamContinuationTokens,
+ Instant parentLowWatermark,
+ List<NewPartition> parentPartitions) {
+ this(partition, changeStreamContinuationTokens, "", parentLowWatermark, parentPartitions, null);
+ }
+
+ public PartitionRecord(
+ ByteStringRange partition,
+ Instant startTime,
+ Instant parentLowWatermark,
+ List<NewPartition> parentPartitions) {
+ this(partition, startTime, "", parentLowWatermark, parentPartitions, null);
+ }
public PartitionRecord(
ByteStringRange partition,
Instant startTime,
String uuid,
Instant parentLowWatermark,
+ List<NewPartition> parentPartitions,
@Nullable Instant endTime) {
this.partition = partition;
this.startTime = startTime;
this.uuid = uuid;
this.parentLowWatermark = parentLowWatermark;
this.endTime = endTime;
- }
-
- @Nullable
- public Instant getEndTime() {
- return endTime;
- }
-
- public void setEndTime(@Nullable Instant endTime) {
- this.endTime = endTime;
+ this.parentPartitions = parentPartitions;
}
public PartitionRecord(
@@ -71,12 +81,14 @@ public class PartitionRecord implements Serializable {
List<ChangeStreamContinuationToken> changeStreamContinuationTokens,
String uuid,
Instant parentLowWatermark,
+ List<NewPartition> parentPartitions,
@Nullable Instant endTime) {
this.partition = partition;
this.changeStreamContinuationTokens = changeStreamContinuationTokens;
this.uuid = uuid;
this.parentLowWatermark = parentLowWatermark;
this.endTime = endTime;
+ this.parentPartitions = parentPartitions;
}
@Nullable
@@ -84,42 +96,38 @@ public class PartitionRecord implements Serializable {
return startTime;
}
- public void setStartTime(@Nullable Instant startTime) {
- this.startTime = startTime;
- }
-
public String getUuid() {
return uuid;
}
- public void setUuid(String uuid) {
- this.uuid = uuid;
- }
-
public Instant getParentLowWatermark() {
return parentLowWatermark;
}
- public void setParentLowWatermark(Instant parentLowWatermark) {
- this.parentLowWatermark = parentLowWatermark;
- }
-
public ByteStringRange getPartition() {
return partition;
}
- public void setPartition(ByteStringRange partition) {
- this.partition = partition;
- }
-
@Nullable
public List<ChangeStreamContinuationToken> getChangeStreamContinuationTokens() {
return changeStreamContinuationTokens;
}
- public void setChangeStreamContinuationTokens(
- @Nullable List<ChangeStreamContinuationToken> changeStreamContinuationTokens) {
- this.changeStreamContinuationTokens = changeStreamContinuationTokens;
+ @Nullable
+ public Instant getEndTime() {
+ return endTime;
+ }
+
+ public List<NewPartition> getParentPartitions() {
+ return parentPartitions;
+ }
+
+ public void setUuid(String uuid) {
+ this.uuid = uuid;
+ }
+
+ public void setEndTime(@Nullable Instant endTime) {
+ this.endTime = endTime;
}
@Override
@@ -131,13 +139,14 @@ public class PartitionRecord implements Serializable {
return false;
}
PartitionRecord that = (PartitionRecord) o;
- return getPartition().equals(that.getPartition())
+ return Objects.equals(getPartition(), that.getPartition())
&& Objects.equals(getStartTime(), that.getStartTime())
&& Objects.equals(
getChangeStreamContinuationTokens(), that.getChangeStreamContinuationTokens())
&& Objects.equals(getEndTime(), that.getEndTime())
- && getUuid().equals(that.getUuid())
- && Objects.equals(getParentLowWatermark(), that.getParentLowWatermark());
+ && Objects.equals(getUuid(), that.getUuid())
+ && Objects.equals(getParentLowWatermark(), that.getParentLowWatermark())
+ && Objects.equals(parentPartitions, that.parentPartitions);
}
@Override
@@ -148,7 +157,8 @@ public class PartitionRecord implements Serializable {
getChangeStreamContinuationTokens(),
getEndTime(),
getUuid(),
- getParentLowWatermark());
+ getParentLowWatermark(),
+ parentPartitions);
}
@Override
@@ -160,11 +170,15 @@ public class PartitionRecord implements Serializable {
+ startTime
+ ", changeStreamContinuationTokens="
+ changeStreamContinuationTokens
+ + ", endTime="
+ + endTime
+ ", uuid='"
+ uuid
+ '\''
+ ", parentLowWatermark="
+ parentLowWatermark
+ + ", parentPartitions="
+ + parentPartitions
+ '}';
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/StreamPartitionWithWatermark.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/StreamPartitionWithWatermark.java
new file mode 100644
index 00000000000..7562d9b5dd4
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/StreamPartitionWithWatermark.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.model;
+
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
+
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.io.Serializable;
+import java.util.Objects;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
+import org.joda.time.Instant;
+
+@Internal
+public class StreamPartitionWithWatermark implements Serializable {
+ private static final long serialVersionUID = -318960470803696894L;
+
+ final ByteStringRange partition;
+ final Instant watermark;
+
+ public StreamPartitionWithWatermark(ByteStringRange partition, Instant watermark) {
+ this.partition = partition;
+ this.watermark = watermark;
+ }
+
+ public ByteStringRange getPartition() {
+ return partition;
+ }
+
+ public Instant getWatermark() {
+ return watermark;
+ }
+
+ @Override
+ public boolean equals(@Nullable Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof StreamPartitionWithWatermark)) {
+ return false;
+ }
+ StreamPartitionWithWatermark that = (StreamPartitionWithWatermark) o;
+ return Objects.equals(getPartition(), that.getPartition())
+ && Objects.equals(getWatermark(), that.getWatermark());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getPartition(), getWatermark());
+ }
+
+ @Override
+ public String toString() {
+ return "StreamPartitionWithWatermark{"
+ + "partition="
+ + formatByteStringRange(partition)
+ + ", watermark="
+ + watermark
+ + '}';
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleaner.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleaner.java
new file mode 100644
index 00000000000..cd45306eee9
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleaner.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler;
+
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.doPartitionsOverlap;
+
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.joda.time.Duration;
+
+@Internal
+public class OrphanedMetadataCleaner {
+ private static final Duration ORPHANED_NEW_PARTITION_WAIT = Duration.standardMinutes(15);
+ List<NewPartition> newPartitions = new ArrayList<>();
+ @Nullable List<ByteStringRange> missingPartitions = null;
+
+ /**
+ * Add NewPartition if it hasn't been updated for 15 minutes.
+ *
+ * @param newPartition new partition to clean up.
+ */
+ public void addIncompleteNewPartitions(NewPartition newPartition) {
+ if (newPartition.getLastUpdated().plus(ORPHANED_NEW_PARTITION_WAIT).isBeforeNow()) {
+ this.newPartitions.add(newPartition);
+ }
+ }
+
+ /**
+ * Add all the missingPartitions. This must be called in order evaluate orphaned new partition.
+ *
+ * @param missingPartitions add missingPartitions.
+ */
+ public void addMissingPartitions(List<ByteStringRange> missingPartitions) {
+ if (this.missingPartitions == null) {
+ this.missingPartitions = new ArrayList<>();
+ }
+ this.missingPartitions.addAll(missingPartitions);
+ }
+
+ /**
+ * Returns a list of NewPartition that have been around for a while and do not overlap with any
+ * missing partition.
+ *
+ * <p>Must call <code>addMissingPartitions</code> before otherwise no orphan can be cleaned up.
+ *
+ * @return NewPartitions safe to clean up.
+ */
+ public List<NewPartition> getOrphanedNewPartitions() {
+ List<NewPartition> orphanedNewPartitions = new ArrayList<>();
+ List<ByteStringRange> missingPartitions = this.missingPartitions;
+ if (missingPartitions == null) {
+ return orphanedNewPartitions;
+ }
+ for (NewPartition newPartition : this.newPartitions) {
+ boolean hasOverlap = false;
+ for (ByteStringRange missingPartition : missingPartitions) {
+ if (doPartitionsOverlap(newPartition.getPartition(), missingPartition)) {
+ hasOverlap = true;
+ break;
+ }
+ }
+ if (!hasOverlap) {
+ orphanedNewPartitions.add(newPartition);
+ }
+ }
+ return orphanedNewPartitions;
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java
index 931e8a6d88f..41e07081d69 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java
@@ -17,18 +17,26 @@
*/
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.coverSameKeySpace;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.doPartitionsOverlap;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.isValidPartition;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingPartitionsFrom;
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
-import com.google.protobuf.ByteString;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
+import java.util.Map;
import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.joda.time.Duration;
import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* There can be a race when many splits and merges happen to a single partition in quick succession.
@@ -58,15 +66,20 @@ import org.joda.time.Instant;
*/
@Internal
public class PartitionReconciler {
- HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile = new HashMap<>();
- HashMap<ByteStringRange, ByteString> newPartitions = new HashMap<>();
- MetadataTableDao metadataTableDao;
+ private static final Logger LOG = LoggerFactory.getLogger(PartitionReconciler.class);
+
+ private HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+ private final List<NewPartition> newPartitions = new ArrayList<>();
+ private final MetadataTableDao metadataTableDao;
+ private final ChangeStreamMetrics metrics;
// The amount of delay allowed before we consider a partition to be probably missing.
- private static final long MISSING_PARTITION_DELAY_MILLI = 5 * 60 * 1000L;
+ private static final Duration MISSING_PARTITION_SHORT_DELAY = Duration.standardMinutes(1);
+ private static final Duration MISSING_PARTITION_LONG_DELAY = Duration.standardMinutes(10);
- public PartitionReconciler(MetadataTableDao metadataTableDao) {
+ public PartitionReconciler(MetadataTableDao metadataTableDao, ChangeStreamMetrics metrics) {
this.metadataTableDao = metadataTableDao;
+ this.metrics = metrics;
}
/**
@@ -87,67 +100,143 @@ public class PartitionReconciler {
* @param missingPartitions partitions not being streamed.
*/
public void addMissingPartitions(List<ByteStringRange> missingPartitions) {
- HashMap<ByteStringRange, Long> alreadyMissingPartitions =
+ HashMap<ByteStringRange, Instant> alreadyMissingPartitionDurations =
metadataTableDao.readDetectNewPartitionMissingPartitions();
- HashMap<ByteStringRange, Long> missingPartitionDuration = new HashMap<>();
- long now = Instant.now().getMillis();
+ missingPartitionDurations = new HashMap<>();
+ Instant now = Instant.now();
for (ByteStringRange missingPartition : missingPartitions) {
- if (!isValidPartition(missingPartition)) {
- continue;
- }
- if (alreadyMissingPartitions.containsKey(missingPartition)) {
- missingPartitionDuration.put(
- missingPartition, alreadyMissingPartitions.get(missingPartition));
- if (alreadyMissingPartitions.get(missingPartition) + MISSING_PARTITION_DELAY_MILLI < now) {
- partitionsToReconcile.put(missingPartition, new HashSet<>());
- }
- } else {
- missingPartitionDuration.put(missingPartition, now);
- }
+ missingPartitionDurations.put(
+ missingPartition, alreadyMissingPartitionDurations.getOrDefault(missingPartition, now));
}
- metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDuration);
+ metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
}
/**
- * Capture NewPartition row that's waiting to be created. If any of these NewPartition row
+ * Capture NewPartition row that cannot merge on its own. If any of these NewPartition row
* overlaps with partition we notice are missing and needs to be reconciled, we will need to clean
* up these NewPartition to avoid future conflicts and inconsistencies.
*
- * @param partition new partitions waiting to be created.
- * @param rowKey the full row key of the new partition.
+ * @param newPartition new partition waiting to be created.
*/
- public void addNewPartition(ByteStringRange partition, ByteString rowKey) {
- newPartitions.put(partition, rowKey);
+ public void addIncompleteNewPartitions(NewPartition newPartition) {
+ newPartitions.add(newPartition);
}
/**
- * Find overlapping partitions between partitionToReconcile and newPartitions. This is to support
- * the effort of identifying the new partition rows that are stuck and needs to be cleaned up
- * because we have successfully reconciled the problem and created a new partition.
+ * Get a list of single token new partitions that have parent partition overlapping
+ * missingPartitions. This is to support the effort of identifying the new partition rows that are
+ * stuck and needs to be cleaned up because we have successfully reconciled the problem and
+ * created a new partition.
*
- * @param partitionToReconcile partition that will be created
+ * @param missingPartition partition that will be created
* @return a set of new partitions that overlaps with partitionToReconcile
*/
- private Set<ByteString> findOverlappingNewPartitions(ByteStringRange partitionToReconcile) {
- Set<ByteString> overlappingRowKey = new HashSet<>();
- for (ByteStringRange newPartition : newPartitions.keySet()) {
- if (doPartitionsOverlap(newPartition, partitionToReconcile)) {
- overlappingRowKey.add(newPartitions.get(newPartition));
+ private List<NewPartition> findOverlappingNewPartitions(ByteStringRange missingPartition) {
+ // TODO: Possibly precompute a map from parentPartitions to newPartitions on the first call to
+ // this function so we don't have to repeat it.
+ List<NewPartition> overlappingNewPartitions = new ArrayList<>();
+ for (NewPartition newPartition : newPartitions) {
+ for (ByteStringRange parentPartition : newPartition.getParentPartitions()) {
+ if (doPartitionsOverlap(parentPartition, missingPartition)) {
+ // Return a NewPartition that only contains the parentPartition.
+ NewPartition splicedNewPartition =
+ newPartition.getSingleTokenNewPartition(parentPartition);
+ if (splicedNewPartition == null) {
+ continue;
+ }
+ overlappingNewPartitions.add(splicedNewPartition);
+ }
}
}
- return overlappingRowKey;
+ return overlappingNewPartitions;
}
/**
- * Match partitions that have been missing for a while and need to be reconciled with NewPartition
- * row key. Find NewPartition row key that overlaps with the reconciled partitions to clean them
- * up.
+ * For missing partitions, try to organize the mismatched parent tokens in a way to fill the
+ * missing partitions.
+ *
+ * <p>If there are parent tokens that when combined form a missing partition, it can be outputted
+ * as a merge of the missing partition.
+ *
+ * <p>If there are no parent tokens for a missing partition, it will need to be reconciled with
+ * adjusted low watermark. This is a catch-all solution. We don't expect to ever get into this
+ * situation. Missing partitions should all be mismatched merges that can be reconciled by
+ * organizing them correctly.
*
- * @return missing partitions and related NewPartition rows keys to delete.
+ * @param lowWatermark watermark that all reconciled partition should have
+ * @param startTime to help compute optimal reconcile point
+ * @return reconciled PartitionRecord.
*/
- public HashMap<ByteStringRange, Set<ByteString>> getPartitionsToReconcile() {
- partitionsToReconcile.replaceAll((r, v) -> findOverlappingNewPartitions(r));
- return partitionsToReconcile;
+ public List<PartitionRecord> getPartitionsToReconcile(Instant lowWatermark, Instant startTime) {
+ // This value is calculated in case that we reconcile without continuation tokens, we will use
+ // an hour prior to low watermark because low watermark is only an estimate. By reading back 1
+ // hour, it should cover any changes missed. We also want to make sure that the reconcile time
+ // isn't before the start time of the pipeline.
+ Instant reconciledTime = lowWatermark.minus(Duration.standardMinutes(60));
+ if (reconciledTime.compareTo(startTime) < 0) {
+ reconciledTime = startTime;
+ }
+ List<PartitionRecord> reconciledPartitions = new ArrayList<>();
+ for (Map.Entry<ByteStringRange, Instant> partitionDuration :
+ missingPartitionDurations.entrySet()) {
+ // The partition hasn't been missing for even the short duration.
+ if (!partitionDuration.getValue().plus(MISSING_PARTITION_SHORT_DELAY).isBeforeNow()) {
+ continue;
+ }
+ ByteStringRange missingPartition = partitionDuration.getKey();
+ List<NewPartition> overlappingNewPartitions = findOverlappingNewPartitions(missingPartition);
+ List<ByteStringRange> overlappingParentPartitions = new ArrayList<>();
+ for (NewPartition newPartition : overlappingNewPartitions) {
+ overlappingParentPartitions.add(newPartition.getParentPartitions().get(0));
+ }
+ // If the parents are equal to the missing partition, a new partition can be formed.
+ if (coverSameKeySpace(overlappingParentPartitions, missingPartition)) {
+ List<ChangeStreamContinuationToken> allTokens = new ArrayList<>();
+ for (NewPartition newPartition : overlappingNewPartitions) {
+ allTokens.add(newPartition.getChangeStreamContinuationTokens().get(0));
+ }
+
+ metrics.incPartitionReconciledWithTokenCount();
+ PartitionRecord record =
+ new PartitionRecord(
+ missingPartition, allTokens, lowWatermark, overlappingNewPartitions);
+ reconciledPartitions.add(record);
+ continue;
+ }
+ // The parents are not equal to the missing partition. If the missing partition has been
+ // around for more than 10 minutes, it needs to be reconciled.
+ if (partitionDuration.getValue().plus(MISSING_PARTITION_LONG_DELAY).isBeforeNow()) {
+ // Try outputting the parent partitions as their own new partition, so they can get more
+ // recent merge targets.
+ for (NewPartition newPartition : overlappingNewPartitions) {
+ metrics.incPartitionReconciledWithTokenCount();
+ PartitionRecord record =
+ new PartitionRecord(
+ newPartition.getChangeStreamContinuationTokens().get(0).getPartition(),
+ newPartition.getChangeStreamContinuationTokens(),
+ lowWatermark,
+ Collections.singletonList(newPartition));
+ reconciledPartitions.add(record);
+ }
+ // Also output partition not overlapped by new partitions.
+ // Get the missing partition from parentPartitions and missingPartition.
+ List<ByteStringRange> missingPartitionsFromParents =
+ getMissingPartitionsFrom(
+ overlappingParentPartitions,
+ missingPartition.getStart(),
+ missingPartition.getEnd());
+ for (ByteStringRange missing : missingPartitionsFromParents) {
+ // This partition is truly missing. There are no tokens representing this. We restart at
+ // the calculated reconcile time.
+ metrics.incPartitionReconciledWithoutTokenCount();
+ PartitionRecord record =
+ new PartitionRecord(missing, reconciledTime, lowWatermark, Collections.emptyList());
+ reconciledPartitions.add(record);
+ LOG.error("DNP: Reconciling partition because we're missing a token {}", record);
+ }
+ }
+ }
+ return reconciledPartitions;
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTracker.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTracker.java
new file mode 100644
index 00000000000..32f619b37c4
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTracker.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction;
+
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.range.OffsetRange;
+import org.apache.beam.sdk.transforms.splittabledofn.GrowableOffsetRangeTracker;
+import org.apache.beam.sdk.transforms.splittabledofn.SplitResult;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+@Internal
+public class DetectNewPartitionsTracker extends GrowableOffsetRangeTracker {
+
+ public DetectNewPartitionsTracker(long start) {
+ super(start, new UnboundedRangeEndEstimator());
+ }
+
+ @Override
+ // Suppress build error that prevents this from being nullable. For some reason it
+ // believes GrowableOffsetRangeTracker trySplit is NonNull even though it is nullable
+ @SuppressWarnings("override.return")
+ public @Nullable SplitResult<OffsetRange> trySplit(double fractionOfRemainder) {
+ // Refuse to split because we only ever want one active restriction for DNP
+ if (fractionOfRemainder != 0) {
+ return null;
+ }
+ return super.trySplit(fractionOfRemainder);
+ }
+
+ // DNP should always be unbounded so we hardcode Long.MAX_VALUE
+ private static class UnboundedRangeEndEstimator implements RangeEndEstimator {
+
+ @Override
+ public long estimate() {
+ return Long.MAX_VALUE;
+ }
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java
index 57aac5203d0..923f8d3e7bb 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java
@@ -62,11 +62,10 @@ public class ReadChangeStreamPartitionProgressTracker
/**
* This restriction tracker is for unbounded streams.
*
- * @return {@link
- * org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.IsBounded.UNBOUNDED}
+ * @return {@link IsBounded.UNBOUNDED}
*/
@Override
- public RestrictionTracker.IsBounded isBounded() {
+ public IsBounded isBounded() {
return IsBounded.UNBOUNDED;
}
@@ -76,11 +75,11 @@ public class ReadChangeStreamPartitionProgressTracker
* runner initiated checkpoint or the streamProgress contains a closeStream response and not a
* token.
*
- * @throws java.lang.IllegalStateException when the restriction is not done and there is more work
- * to be done.
+ * @throws IllegalStateException when the restriction is not done and there is more work to be
+ * done.
*/
@Override
- public void checkDone() throws java.lang.IllegalStateException {
+ public void checkDone() throws IllegalStateException {
boolean done =
shouldStop || streamProgress.getCloseStream() != null || streamProgress.isFailToLock();
Preconditions.checkState(done, "There's more work to be done");
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java
index 6ffef3790c4..ce2cf11a842 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java
@@ -37,7 +37,7 @@ import org.joda.time.Instant;
*/
@Internal
public class StreamProgress implements Serializable {
- private static final long serialVersionUID = -8597355120329526194L;
+ private static final long serialVersionUID = -5384329262726188695L;
private @Nullable ChangeStreamContinuationToken currentToken;
private @Nullable Instant estimatedLowWatermark;
@@ -76,6 +76,10 @@ public class StreamProgress implements Serializable {
this.failToLock = failToLock;
}
+ public boolean isEmpty() {
+ return closeStream == null && currentToken == null;
+ }
+
@Override
public boolean equals(@Nullable Object o) {
if (this == o) {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelperTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelperTest.java
index 39751230176..943a748908a 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelperTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelperTest.java
@@ -20,7 +20,9 @@ package org.apache.beam.sdk.io.gcp.bigtable.changestreams;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.doPartitionsOverlap;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getIntersectingPartition;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingAndOverlappingPartitionsFromKeySpace;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingPartitionsFrom;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingPartitionsFromEntireKeySpace;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getOverlappingPartitions;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.isValidPartition;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.partitionsToString;
import static org.junit.Assert.assertEquals;
@@ -29,6 +31,7 @@ import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import com.google.protobuf.ByteString;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -40,17 +43,6 @@ import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public class ByteStringRangeHelperTest {
- @Test
- public void testParentIsEntireKeySpaceIsSuperSet() {
- List<ByteStringRange> parentPartitions = new ArrayList<>();
- ByteStringRange partition = ByteStringRange.create("", "");
- parentPartitions.add(partition);
-
- ByteStringRange childPartition = ByteStringRange.create("A", "B");
-
- assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
- }
-
@Test
public void testChildIsEntireKeySpaceParentIsLeftSubSet() {
List<ByteStringRange> parentPartitions = new ArrayList<>();
@@ -58,7 +50,7 @@ public class ByteStringRangeHelperTest {
parentPartitions.add(partition);
ByteStringRange childPartition = ByteStringRange.create("", "");
- assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
@@ -68,11 +60,11 @@ public class ByteStringRangeHelperTest {
parentPartitions.add(partition);
ByteStringRange childPartition = ByteStringRange.create("", "");
- assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
- public void testChildIsEntireKeySpaceParentIsSuperSet() {
+ public void testChildIsEntireKeySpaceParentCoverKeySpace() {
List<ByteStringRange> parentPartitions = new ArrayList<>();
ByteStringRange partition1 = ByteStringRange.create("", "n");
ByteStringRange partition2 = ByteStringRange.create("n", "");
@@ -80,81 +72,81 @@ public class ByteStringRangeHelperTest {
parentPartitions.add(partition2);
ByteStringRange childPartition = ByteStringRange.create("", "");
- assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertTrue(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
- public void testParentKeySpaceStartsBeforeAndEndAfterChildIsSuperSet() {
+ public void testParentKeySpaceStartsBeforeAndEndAfterChildDoesNotCoverKeySpace() {
List<ByteStringRange> parentPartitions = new ArrayList<>();
ByteStringRange partition = ByteStringRange.create("A", "B");
parentPartitions.add(partition);
ByteStringRange childPartition = ByteStringRange.create("AA", "AB");
- assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
- public void testParentStartKeyIsAfterChildStartKeyIsNotSuperSet() {
+ public void testParentStartKeyIsAfterChildStartKeyDoesNotCoverKeySpace() {
List<ByteStringRange> parentPartitions = new ArrayList<>();
ByteStringRange partition = ByteStringRange.create("AA", "B");
parentPartitions.add(partition);
ByteStringRange childPartition = ByteStringRange.create("A", "AB");
- assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
- public void testParentEndKeyIsBeforeChildEndKeyIsNotSuperSet() {
+ public void testParentEndKeyIsBeforeChildEndKeyDoesNotCoverKeySpace() {
List<ByteStringRange> parentPartitions = new ArrayList<>();
ByteStringRange partition = ByteStringRange.create("A", "B");
parentPartitions.add(partition);
ByteStringRange childPartition = ByteStringRange.create("AA", "BA");
- assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
- public void testParentIsSameAsChildIsSuperSet() {
+ public void testParentIsSameAsChildCoverKeySpace() {
List<ByteStringRange> parentPartitions = new ArrayList<>();
ByteStringRange partition = ByteStringRange.create("A", "B");
parentPartitions.add(partition);
ByteStringRange childPartition = ByteStringRange.create("A", "B");
- assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertTrue(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
- public void testParentIsMissingPartitionIsNotSuperSet() {
+ public void testParentIsMissingPartitionDoesNotCoverKeySpace() {
ByteStringRange partition1 = ByteStringRange.create("A", "B");
ByteStringRange partition2 = ByteStringRange.create("C", "Z");
List<ByteStringRange> parentPartitions = Arrays.asList(partition1, partition2);
ByteStringRange childPartition = ByteStringRange.create("A", "Z");
- assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
- public void testParentHasOverlapIsSuperSet() {
+ public void testParentHasOverlapDoesNotCoverKeySpace() {
ByteStringRange partition1 = ByteStringRange.create("A", "C");
ByteStringRange partition2 = ByteStringRange.create("B", "Z");
List<ByteStringRange> parentPartitions = Arrays.asList(partition1, partition2);
ByteStringRange childPartition = ByteStringRange.create("A", "Z");
- assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
- public void testEmptyParentsIsNotSuperset() {
+ public void testEmptyParentsDoesNotCoverKeySpace() {
List<ByteStringRange> parentPartitions = Collections.emptyList();
ByteStringRange childPartition = ByteStringRange.create("", "");
- assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+ assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
}
@Test
@@ -181,81 +173,213 @@ public class ByteStringRangeHelperTest {
}
@Test
- public void testGetMissingAndOverlappingPartitionFromKeySpaceEmptyPartition() {
+ public void testGetMissingPartitionsEmptyPartition() {
List<ByteStringRange> partitions = new ArrayList<>();
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
+ List<ByteStringRange> missingPartitions =
+ getMissingPartitionsFrom(
+ partitions, ByteString.copyFromUtf8("A"), ByteString.copyFromUtf8("B"));
+ assertEquals(Collections.singletonList(ByteStringRange.create("A", "B")), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionFromPartitions() {
+ List<ByteStringRange> partitions =
+ Arrays.asList(ByteStringRange.create("A", "B"), ByteStringRange.create("C", "D"));
+ List<ByteStringRange> missingPartitions =
+ getMissingPartitionsFrom(
+ partitions, ByteString.copyFromUtf8("A"), ByteString.copyFromUtf8("D"));
+ assertEquals(Collections.singletonList(ByteStringRange.create("B", "C")), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromOverlappingStart() {
+ List<ByteStringRange> partitions =
+ Arrays.asList(ByteStringRange.create("A", "C"), ByteStringRange.create("E", "F"));
+ List<ByteStringRange> missingPartitions =
+ getMissingPartitionsFrom(
+ partitions, ByteString.copyFromUtf8("B"), ByteString.copyFromUtf8("G"));
+ assertEquals(
+ Arrays.asList(ByteStringRange.create("C", "E"), ByteStringRange.create("F", "G")),
+ missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromOverlappingEnd() {
+ List<ByteStringRange> partitions =
+ Arrays.asList(ByteStringRange.create("A", "C"), ByteStringRange.create("E", "G"));
+ List<ByteStringRange> missingPartitions =
+ getMissingPartitionsFrom(
+ partitions, ByteString.copyFromUtf8("A"), ByteString.copyFromUtf8("F"));
+ assertEquals(Collections.singletonList(ByteStringRange.create("C", "E")), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromOverlappingPartitions() {
+ List<ByteStringRange> partitions =
+ Arrays.asList(ByteStringRange.create("A", "C"), ByteStringRange.create("B", "G"));
+ List<ByteStringRange> missingPartitions =
+ getMissingPartitionsFrom(
+ partitions, ByteString.copyFromUtf8("A"), ByteString.copyFromUtf8("F"));
+ assertEquals(Collections.emptyList(), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromPartitionsOutsideRange() {
+ List<ByteStringRange> partitions =
+ Arrays.asList(ByteStringRange.create("A", "B"), ByteStringRange.create("C", "D"));
+ List<ByteStringRange> missingPartitions =
+ getMissingPartitionsFrom(
+ partitions, ByteString.copyFromUtf8("G"), ByteString.copyFromUtf8("H"));
+ assertEquals(Collections.singletonList(ByteStringRange.create("G", "H")), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromPartitionsPartiallyOutsideRange() {
+ List<ByteStringRange> partitions =
+ Arrays.asList(ByteStringRange.create("A", "B"), ByteStringRange.create("D", "E"));
+ List<ByteStringRange> missingPartitions =
+ getMissingPartitionsFrom(
+ partitions, ByteString.copyFromUtf8("C"), ByteString.copyFromUtf8("H"));
assertEquals(
- Collections.singletonList(ByteStringRange.create("", "")), missingAndOverlappingPartitions);
+ Arrays.asList(ByteStringRange.create("C", "D"), ByteStringRange.create("E", "H")),
+ missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromKeySpaceEmptyPartition() {
+ List<ByteStringRange> partitions = new ArrayList<>();
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.singletonList(ByteStringRange.create("", "")), missingPartitions);
}
@Test
- public void testGetMissingAndOverlappingPartitionFromKeySpaceSinglePartition() {
+ public void testGetMissingPartitionsFromKeySpaceSinglePartition() {
ByteStringRange partition1 = ByteStringRange.create("", "");
List<ByteStringRange> partitions = Collections.singletonList(partition1);
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
- assertEquals(Collections.emptyList(), missingAndOverlappingPartitions);
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.emptyList(), missingPartitions);
}
@Test
- public void testGetMissingAndOverlappingPartitionFromKeySpaceNoMissingPartition() {
+ public void testGetMissingPartitionsFromKeySpaceNoMissingPartition() {
ByteStringRange partition1 = ByteStringRange.create("", "A");
ByteStringRange partition2 = ByteStringRange.create("A", "B");
ByteStringRange partition3 = ByteStringRange.create("B", "");
List<ByteStringRange> partitions = Arrays.asList(partition1, partition2, partition3);
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
- assertEquals(Collections.emptyList(), missingAndOverlappingPartitions);
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.emptyList(), missingPartitions);
}
@Test
- public void testGetMissingAndOverlappingPartitionFromKeySpaceMissingStartPartition() {
+ public void testGetMissingPartitionsFromKeySpaceMissingStartPartition() {
ByteStringRange partition1 = ByteStringRange.create("", "A");
ByteStringRange partition2 = ByteStringRange.create("A", "B");
ByteStringRange partition3 = ByteStringRange.create("B", "");
List<ByteStringRange> partitions = Arrays.asList(partition2, partition3);
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
- assertEquals(Collections.singletonList(partition1), missingAndOverlappingPartitions);
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.singletonList(partition1), missingPartitions);
}
@Test
- public void testGetMissingAndOverlappingPartitionFromKeySpaceMissingEndPartition() {
+ public void testGetMissingPartitionsFromKeySpaceMissingEndPartition() {
ByteStringRange partition1 = ByteStringRange.create("", "A");
ByteStringRange partition2 = ByteStringRange.create("A", "B");
ByteStringRange partition3 = ByteStringRange.create("B", "");
List<ByteStringRange> partitions = Arrays.asList(partition1, partition2);
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
- assertEquals(Collections.singletonList(partition3), missingAndOverlappingPartitions);
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.singletonList(partition3), missingPartitions);
}
@Test
- public void testGetMissingAndOverlappingPartitionFromKeySpaceMissingMiddlePartition() {
+ public void testGetMissingPartitionsFromKeySpaceMissingMiddlePartition() {
ByteStringRange partition1 = ByteStringRange.create("", "A");
ByteStringRange partition2 = ByteStringRange.create("A", "B");
ByteStringRange partition3 = ByteStringRange.create("B", "");
List<ByteStringRange> partitions = Arrays.asList(partition1, partition3);
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
- assertEquals(Collections.singletonList(partition2), missingAndOverlappingPartitions);
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.singletonList(partition2), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromKeySpaceOverlapPartition() {
+ ByteStringRange partition1 = ByteStringRange.create("", "B");
+ ByteStringRange partition2 = ByteStringRange.create("A", "");
+ List<ByteStringRange> partitions = Arrays.asList(partition1, partition2);
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.emptyList(), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromKeySpaceOverlapAndMissingPartition() {
+ ByteStringRange partition1 = ByteStringRange.create("", "B");
+ ByteStringRange partition2 = ByteStringRange.create("C", "D");
+ ByteStringRange partition3 = ByteStringRange.create("A", "C");
+ ByteStringRange partition4 = ByteStringRange.create("E", "");
+ ByteStringRange partition5 = ByteStringRange.create("C", "E");
+ List<ByteStringRange> partitions =
+ Arrays.asList(partition1, partition2, partition3, partition4, partition5);
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.emptyList(), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromKeySpaceOverlapWithOpenEndKey() {
+ ByteStringRange fullKeySpace = ByteStringRange.create("", "");
+ ByteStringRange partialKeySpace = ByteStringRange.create("n", "");
+ List<ByteStringRange> partitions = Arrays.asList(fullKeySpace, partialKeySpace);
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.emptyList(), missingPartitions);
+ }
+
+ @Test
+ public void testGetMissingPartitionsFromEntireKeySpaceWithOverlappingPartitions() {
+ List<ByteStringRange> partitions =
+ Arrays.asList(
+ ByteStringRange.create("", "A"),
+ ByteStringRange.create("B", "C"),
+ ByteStringRange.create("A", "D"),
+ ByteStringRange.create("D", ""));
+ List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+ assertEquals(Collections.emptyList(), missingPartitions);
+ }
+
+ @Test
+ public void testGetOverlappingPartitionsFromKeySpaceEmptyPartition() {
+ List<ByteStringRange> partitions = new ArrayList<>();
+ List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
+ assertEquals(Collections.emptyList(), overlappingPartitions);
+ }
+
+ @Test
+ public void testGetOverlappingPartitionsSinglePartition() {
+ ByteStringRange partition1 = ByteStringRange.create("", "");
+ List<ByteStringRange> partitions = Collections.singletonList(partition1);
+ List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
+ assertEquals(Collections.emptyList(), overlappingPartitions);
+ }
+
+ @Test
+ public void testGetOverlappingPartitionsNoMissingPartition() {
+ ByteStringRange partition1 = ByteStringRange.create("", "A");
+ ByteStringRange partition2 = ByteStringRange.create("A", "B");
+ ByteStringRange partition3 = ByteStringRange.create("B", "");
+ List<ByteStringRange> partitions = Arrays.asList(partition1, partition2, partition3);
+ List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
+ assertEquals(Collections.emptyList(), overlappingPartitions);
}
@Test
- public void testGetMissingAndOverlappingPartitionFromKeySpaceOverlapPartition() {
+ public void testGetOverlappingPartitionsOverlapPartition() {
ByteStringRange partition1 = ByteStringRange.create("", "B");
ByteStringRange partition2 = ByteStringRange.create("A", "");
List<ByteStringRange> partitions = Arrays.asList(partition1, partition2);
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
+ List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
assertEquals(
- Collections.singletonList(ByteStringRange.create("B", "A")),
- missingAndOverlappingPartitions);
+ Collections.singletonList(ByteStringRange.create("A", "B")), overlappingPartitions);
}
@Test
- public void testGetMissingAndOverlappingPartitionFromKeySpaceOverlapAndMissingPartition() {
+ public void testGetOverlappingPartitionsOverlapAndMissingPartition() {
ByteStringRange partition1 = ByteStringRange.create("", "B");
ByteStringRange partition2 = ByteStringRange.create("C", "D");
ByteStringRange partition3 = ByteStringRange.create("A", "C");
@@ -263,25 +387,19 @@ public class ByteStringRangeHelperTest {
ByteStringRange partition5 = ByteStringRange.create("C", "E");
List<ByteStringRange> partitions =
Arrays.asList(partition1, partition2, partition3, partition4, partition5);
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
+ List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
assertEquals(
- Arrays.asList(ByteStringRange.create("B", "A"), ByteStringRange.create("D", "C")),
- missingAndOverlappingPartitions);
+ Arrays.asList(ByteStringRange.create("A", "B"), ByteStringRange.create("C", "D")),
+ overlappingPartitions);
}
@Test
- public void testGetMissingAndOverlappingPartitionsFromKeySpaceOverlapWithOpenEndKey() {
+ public void testGetOverlappingPartitions() {
ByteStringRange fullKeySpace = ByteStringRange.create("", "");
ByteStringRange partialKeySpace = ByteStringRange.create("n", "");
List<ByteStringRange> partitions = Arrays.asList(fullKeySpace, partialKeySpace);
- // TODO come up with a better way to differentiate missing with start key "" and overlapping
- // with end key ""
- ByteStringRange overlappingPartition = ByteStringRange.create("", "n");
- List<ByteStringRange> expectedOverlapping = Collections.singletonList(overlappingPartition);
- List<ByteStringRange> missingAndOverlappingPartitions =
- getMissingAndOverlappingPartitionsFromKeySpace(partitions);
- assertEquals(expectedOverlapping, missingAndOverlappingPartitions);
+ List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
+ assertEquals(Collections.singletonList(ByteStringRange.create("n", "")), overlappingPartitions);
}
@Test
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverterTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverterTest.java
index 5c8b916ffc4..08b85980263 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverterTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverterTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.beam.sdk.io.gcp.bigtable.changestreams;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.microsecondToInstant;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.toJodaTime;
import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.toThreetenInstant;
import static org.junit.Assert.assertEquals;
@@ -41,4 +42,9 @@ public class TimestampConverterTest {
assertEquals(1, TimestampConverter.toSeconds(org.joda.time.Instant.ofEpochSecond(1)));
assertEquals(1000, TimestampConverter.toSeconds(org.joda.time.Instant.ofEpochSecond(1000)));
}
+
+ @Test
+ public void testMicrosecondToJodaInstant() {
+ assertEquals(org.joda.time.Instant.ofEpochMilli(1_234L), microsecondToInstant(1_234_567L));
+ }
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java
index 7724dad1209..6fe272009cc 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java
@@ -123,8 +123,8 @@ public class ChangeStreamActionTest {
public void testChangeStreamMutationUser() {
ByteStringRange partition = ByteStringRange.create("", "");
when(partitionRecord.getPartition()).thenReturn(partition);
- final Instant commitTimestamp = Instant.ofEpochSecond(1000);
- final Instant lowWatermark = Instant.ofEpochSecond(500);
+ final Instant commitTimestamp = Instant.ofEpochMilli(1_000L);
+ final Instant lowWatermark = Instant.ofEpochMilli(500L);
ChangeStreamContinuationToken changeStreamContinuationToken =
ChangeStreamContinuationToken.create(ByteStringRange.create("", ""), "1234");
ChangeStreamMutation changeStreamMutation = Mockito.mock(ChangeStreamMutation.class);
@@ -155,8 +155,8 @@ public class ChangeStreamActionTest {
public void testChangeStreamMutationGc() {
ByteStringRange partition = ByteStringRange.create("", "");
when(partitionRecord.getPartition()).thenReturn(partition);
- final Instant commitTimestamp = Instant.ofEpochSecond(1000);
- final Instant lowWatermark = Instant.ofEpochSecond(500);
+ final Instant commitTimestamp = Instant.ofEpochMilli(1_000L);
+ final Instant lowWatermark = Instant.ofEpochMilli(500L);
ChangeStreamContinuationToken changeStreamContinuationToken =
ChangeStreamContinuationToken.create(ByteStringRange.create("", ""), "1234");
ChangeStreamMutation changeStreamMutation = Mockito.mock(ChangeStreamMutation.class);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java
index a3d741dcdd8..9d8aee40ed3 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java
@@ -20,6 +20,8 @@ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.never;
@@ -33,9 +35,11 @@ import com.google.cloud.bigtable.data.v2.BigtableDataClient;
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;
@@ -43,12 +47,11 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
import org.apache.beam.sdk.io.range.OffsetRange;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.BundleFinalizer;
-import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators;
@@ -72,17 +75,16 @@ public class DetectNewPartitionsActionTest {
@Mock private ChangeStreamMetrics metrics;
@Mock private GenerateInitialPartitionsAction generateInitialPartitionsAction;
+ @Mock private ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction;
private DetectNewPartitionsAction action;
@Mock private RestrictionTracker<OffsetRange, Long> tracker;
- @Mock private OutputReceiver<PartitionRecord> receiver;
- @Mock private BundleFinalizer bundleFinalizer;
+ @Mock private DoFn.OutputReceiver<PartitionRecord> receiver;
private MetadataTableDao metadataTableDao;
private ManualWatermarkEstimator<Instant> watermarkEstimator;
private Instant startTime;
- private Instant partitionTime;
private Instant endTime;
private static BigtableDataClient dataClient;
private static BigtableTableAdminClient adminClient;
@@ -120,125 +122,407 @@ public class DetectNewPartitionsActionTest {
startTime = Instant.now();
endTime = startTime.plus(Duration.standardSeconds(10));
- partitionTime = startTime.plus(Duration.standardSeconds(10));
+ ProcessNewPartitionsAction processNewPartitionsAction =
+ new ProcessNewPartitionsAction(metrics, metadataTableDao, endTime);
action =
new DetectNewPartitionsAction(
- metrics, metadataTableDao, endTime, generateInitialPartitionsAction);
+ metrics,
+ metadataTableDao,
+ endTime,
+ generateInitialPartitionsAction,
+ resumeFromPreviousPipelineAction,
+ processNewPartitionsAction);
watermarkEstimator = new WatermarkEstimators.Manual(startTime);
}
@Test
- public void testInitialPartitions() throws Exception {
- OffsetRange offsetRange = new OffsetRange(0, Long.MAX_VALUE);
+ public void testUpdateWatermarkAfterCheckpoint() throws Exception {
+ Instant watermark = endTime;
+ OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
+ when(tracker.currentRestriction()).thenReturn(offsetRange);
+ when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+
+ // Watermark estimator is not updated before a checkpoint.
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+ // Update the watermark in the metadata table. This run terminates because watermark == endTime.
+ metadataTableDao.updateDetectNewPartitionWatermark(watermark);
+ // Watermark estimator will be updated with the watermark from the metadata table.
+ assertEquals(
+ DoFn.ProcessContinuation.stop(),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(watermark, watermarkEstimator.currentWatermark());
+ assertEquals(watermark, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+ }
+
+ @Test
+ public void testUpdateWatermarkOnEvenCountAfter10Seconds() throws Exception {
+ // We update watermark every 2 iterations only if it's been more than 10s since the last update.
+ OffsetRange offsetRange = new OffsetRange(2, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
+ when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+ // Manually set the watermark of DNP to start time with a timestamp of 10s prior.
+ RowMutation rowMutation =
+ RowMutation.create(
+ MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME,
+ metadataTableDao
+ .getChangeStreamNamePrefix()
+ .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX))
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK,
+ MetadataTableAdminDao.QUALIFIER_DEFAULT,
+ Instant.now().minus(Duration.standardSeconds(10)).getMillis() * 1000L,
+ startTime.getMillis());
+ dataClient.mutateRow(rowMutation);
+
+ // Create a partition covering the entire keyspace with watermark after endTime.
+ ByteStringRange partition1 = ByteStringRange.create("", "");
+ Instant watermark1 = endTime.plus(Duration.millis(100));
+ PartitionRecord partitionRecord1 =
+ new PartitionRecord(
+ partition1,
+ watermark1,
+ UniqueIdGenerator.getNextId(),
+ watermark1,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord1);
- when(generateInitialPartitionsAction.run(receiver, tracker, watermarkEstimator, startTime))
- .thenReturn(ProcessContinuation.resume());
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertEquals(watermark1, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+ // On the 2nd run, watermark estimator is updated which is beyond endTime and terminates.
assertEquals(
- ProcessContinuation.resume(),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+ DoFn.ProcessContinuation.stop(),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ verify(tracker, times(1)).tryClaim(offsetRange.getTo());
+ assertEquals(watermark1, watermarkEstimator.currentWatermark());
}
- // Every 10 tryClaim, DNP updates the watermark based on the watermark of all the RCSP.
+ @Test
+ public void testDoNotUpdateWatermarkLessThan10s() throws Exception {
+ // We update watermark every 2 iterations only if it's been more than 10s since the last update.
+ OffsetRange offsetRange = new OffsetRange(2, Long.MAX_VALUE);
+ when(tracker.currentRestriction()).thenReturn(offsetRange);
+ when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+ // Manually set the watermark of DNP to start time with a timestamp of 1s prior.
+ RowMutation rowMutation =
+ RowMutation.create(
+ MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME,
+ metadataTableDao
+ .getChangeStreamNamePrefix()
+ .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX))
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK,
+ MetadataTableAdminDao.QUALIFIER_DEFAULT,
+ Instant.now().minus(Duration.standardSeconds(1)).getMillis() * 1000L,
+ startTime.getMillis());
+ dataClient.mutateRow(rowMutation);
+
+ // Create a partition covering the entire keyspace with watermark after endTime.
+ ByteStringRange partition1 = ByteStringRange.create("", "");
+ Instant watermark1 = endTime.plus(Duration.millis(100));
+ PartitionRecord partitionRecord1 =
+ new PartitionRecord(
+ partition1,
+ watermark1,
+ UniqueIdGenerator.getNextId(),
+ watermark1,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord1);
+
+ // Watermark doesn't get updated because the last time watermark was updated was less than 10s.
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertEquals(startTime, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+
+ // On the 2nd run, watermark estimator is still the same since watermark isn't updated.
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ }
+
+ @Test
+ public void testDoNotUpdateWatermarkOnOddCount() throws Exception {
+ // We only update watermark on even count.
+ OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
+ when(tracker.currentRestriction()).thenReturn(offsetRange);
+ when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+ // Manually set the watermark of DNP to start time with a timestamp of 10s prior.
+ RowMutation rowMutation =
+ RowMutation.create(
+ MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME,
+ metadataTableDao
+ .getChangeStreamNamePrefix()
+ .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX))
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK,
+ MetadataTableAdminDao.QUALIFIER_DEFAULT,
+ Instant.now().minus(Duration.standardSeconds(10)).getMillis() * 1000L,
+ startTime.getMillis());
+ dataClient.mutateRow(rowMutation);
+
+ // Create a partition covering the entire keyspace with watermark after endTime.
+ ByteStringRange partition1 = ByteStringRange.create("", "");
+ Instant watermark1 = endTime.plus(Duration.millis(100));
+ PartitionRecord partitionRecord1 =
+ new PartitionRecord(
+ partition1,
+ watermark1,
+ UniqueIdGenerator.getNextId(),
+ watermark1,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord1);
+
+ // It's been more than 10s since last update but the count is odd.
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertEquals(startTime, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+
+ // On the 2nd run, watermark estimator is still the same since watermark isn't updated.
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ }
+
+ // Every 2 tryClaim, DNP updates the watermark based on the watermark of all the RCSP.
@Test
public void testAdvanceWatermarkWithAllPartitions() throws Exception {
- // We advance watermark on every 10 restriction tracker advancement
+ // We advance watermark on every 2 restriction tracker advancement
OffsetRange offsetRange = new OffsetRange(10, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
+ when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+ when(tracker.tryClaim(offsetRange.getTo())).thenReturn(true);
assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertNull(metadataTableDao.readDetectNewPartitionsState());
// Write 2 partitions to the table that covers entire keyspace.
ByteStringRange partition1 = ByteStringRange.create("", "b");
- Instant watermark1 = partitionTime.plus(Duration.millis(100));
- metadataTableDao.updateWatermark(partition1, watermark1, null);
+ Instant watermark1 = endTime.plus(Duration.millis(100));
+ PartitionRecord partitionRecord1 =
+ new PartitionRecord(
+ partition1,
+ watermark1,
+ UniqueIdGenerator.getNextId(),
+ watermark1,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord1);
ByteStringRange partition2 = ByteStringRange.create("b", "");
Instant watermark2 = endTime.plus(Duration.millis(1));
- metadataTableDao.updateWatermark(partition2, watermark2, null);
+ PartitionRecord partitionRecord2 =
+ new PartitionRecord(
+ partition2,
+ watermark2,
+ UniqueIdGenerator.getNextId(),
+ watermark2,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord2);
+
+ // Updating watermark does not affect the watermark estimator of this run.
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ verify(tracker, times(1)).tryClaim(offsetRange.getFrom());
+
+ // Because the 2 partitions cover the entire keyspace, the watermark should have advanced.
+ // Also note the watermark is watermark2 which is the lowest of the 2 watermarks. Watermark
+ // estimator isn't updated because we update the watermark estimator on the start of the run. We
+ // do update the metadata table with the new watermark value.
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertEquals(watermark2, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+ // On the 2nd run, watermark estimator is updated which is beyond endTime and terminates.
assertEquals(
DoFn.ProcessContinuation.stop(),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
- // TryClaim should be called with the end value because the watermark should be updated to
- // beyond endTime causing the pipeline to terminate.
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
verify(tracker, times(1)).tryClaim(offsetRange.getTo());
-
- // Because the 2 partitions cover the entire keyspace, the watermark should have advanced.
- // Also note the watermark is watermark2 which is the lowest of the 2 watermarks.
assertEquals(watermark2, watermarkEstimator.currentWatermark());
}
- // Every 10 tryClaim, DNP only updates its watermark if all the RCSP currently streamed covers the
+ // Every 2 tryClaim, DNP only updates its watermark if all the RCSP currently streamed covers the
// entire key space. If there's any missing, they are in the process of split or merge. If the
// watermark is updated with missing partitions, the watermark might be further ahead than it
// actually is.
@Test
public void testAdvanceWatermarkWithMissingPartitions() throws Exception {
- // We advance watermark on every 10 restriction tracker advancement
- OffsetRange offsetRange = new OffsetRange(10, Long.MAX_VALUE);
+ // We advance watermark on every 2 restriction tracker advancement
+ OffsetRange offsetRange = new OffsetRange(2, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
- // We return false here, so we do need to test any other functionality other than updating the
- // watermark.
- when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(false);
+ when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertNull(metadataTableDao.readDetectNewPartitionsState());
// Write 2 partitions to the table that DO NOT cover the entire keyspace.
ByteStringRange partition1 = ByteStringRange.create("", "b");
- Instant watermark1 = partitionTime.plus(Duration.millis(100));
+ Instant watermark1 = endTime.plus(Duration.millis(100));
metadataTableDao.updateWatermark(partition1, watermark1, null);
ByteStringRange partition2 = ByteStringRange.create("b", "c");
Instant watermark2 = endTime.plus(Duration.millis(1));
metadataTableDao.updateWatermark(partition2, watermark2, null);
assertEquals(
- DoFn.ProcessContinuation.stop(),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
- // Ensure that we did indeed try to claim the start of offset range because that means watermark
- // was not updated beyond endTime.
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
verify(tracker, times(1)).tryClaim(offsetRange.getFrom());
// Because the 2 partitions DO NOT cover the entire keyspace, watermark stays at startTime.
assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertNull(metadataTableDao.readDetectNewPartitionsState());
+ }
+
+ @Test
+ public void testAdvanceWatermarkWithNewPartitions() throws Exception {
+ // We advance watermark on every 2 restriction tracker advancement
+ OffsetRange offsetRange = new OffsetRange(2, Long.MAX_VALUE);
+ when(tracker.currentRestriction()).thenReturn(offsetRange);
+ when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+ when(tracker.tryClaim(offsetRange.getTo())).thenReturn(true);
+
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+ // Write 2 partitions to the table that DO NOT cover the entire keyspace.
+ ByteStringRange partition1 = ByteStringRange.create("", "b");
+ Instant watermark1 = endTime.plus(Duration.millis(100));
+ PartitionRecord partitionRecord1 =
+ new PartitionRecord(
+ partition1,
+ watermark1,
+ UniqueIdGenerator.getNextId(),
+ watermark1,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord1);
+ ByteStringRange partition2 = ByteStringRange.create("b", "c");
+ Instant watermark2 = endTime.plus(Duration.millis(1));
+ PartitionRecord partitionRecord2 =
+ new PartitionRecord(
+ partition2,
+ watermark2,
+ UniqueIdGenerator.getNextId(),
+ watermark2,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord2);
+
+ ByteStringRange partition3 = ByteStringRange.create("c", "");
+ ChangeStreamContinuationToken token = ChangeStreamContinuationToken.create(partition3, "token");
+ NewPartition newPartition =
+ new NewPartition(partition3, Collections.singletonList(token), watermark2);
+ metadataTableDao.writeNewPartition(newPartition);
+
+ // Updating watermark does not affect the result of this run.
+ assertEquals(
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ verify(tracker, times(1)).tryClaim(offsetRange.getFrom());
+
+ // Because the StreamPartition and NewPartition cover the entire keyspace, the watermark should
+ // have advanced. Also note the watermark is watermark2 which is the lowest watermark. Watermark
+ // estimator isn't updated because we update the watermark estimator on the start of the run. We
+ // do update the metadata table with the new watermark value.
+ assertEquals(startTime, watermarkEstimator.currentWatermark());
+ assertEquals(watermark2, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+
+ // On the 2nd run, watermark estimator is updated which is beyond endTime and terminates.
+ assertEquals(
+ DoFn.ProcessContinuation.stop(),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ verify(tracker, times(1)).tryClaim(offsetRange.getTo());
+ assertEquals(watermark2, watermarkEstimator.currentWatermark());
}
@Test
public void testProcessSplitNewPartitions() throws Exception {
- // Avoid 0 and multiples of 10 so that we can specifically test just reading new partitions.
+ // Avoid 0 and multiples of 2 so that we can specifically test just reading new partitions.
OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
- // ["", "") splits into ["", "a"), ["a","c"), ["c", "")
- ByteStringRange parentPartition = ByteStringRange.create("", "");
+ // ["", "") splits into ["", "a"), ["a","k"), ["k", "")
Instant watermark = Instant.now();
- ByteStringRange child1 = ByteStringRange.create("", "a");
- metadataTableDao.writeNewPartition(
- child1,
+ ByteStringRange childEmptyA = ByteStringRange.create("", "a");
+ ChangeStreamContinuationToken tokenEmptyA =
ChangeStreamContinuationToken.create(
- child1, ByteStringRange.serializeToByteString(child1).toStringUtf8()),
- parentPartition,
- watermark);
- ByteStringRange child2 = ByteStringRange.create("k", "");
- metadataTableDao.writeNewPartition(
- child2,
+ childEmptyA, ByteStringRange.serializeToByteString(childEmptyA).toStringUtf8());
+ NewPartition newPartitionEmptyA =
+ new NewPartition(childEmptyA, Collections.singletonList(tokenEmptyA), watermark);
+ metadataTableDao.writeNewPartition(newPartitionEmptyA);
+ ByteStringRange childKEmpty = ByteStringRange.create("k", "");
+ ChangeStreamContinuationToken tokenKEmpty =
ChangeStreamContinuationToken.create(
- child2, ByteStringRange.serializeToByteString(child2).toStringUtf8()),
- parentPartition,
- watermark);
- ByteStringRange child3 = ByteStringRange.create("a", "k");
- metadataTableDao.writeNewPartition(
- child3,
+ childKEmpty, ByteStringRange.serializeToByteString(childKEmpty).toStringUtf8());
+ NewPartition newPartitionKEmpty =
+ new NewPartition(childKEmpty, Collections.singletonList(tokenKEmpty), watermark);
+ metadataTableDao.writeNewPartition(newPartitionKEmpty);
+ ByteStringRange childAK = ByteStringRange.create("a", "k");
+ ChangeStreamContinuationToken tokenAK =
ChangeStreamContinuationToken.create(
- child3, ByteStringRange.serializeToByteString(child3).toStringUtf8()),
- parentPartition,
- watermark);
+ childAK, ByteStringRange.serializeToByteString(childAK).toStringUtf8());
+ NewPartition newPartitionAK =
+ new NewPartition(childAK, Collections.singletonList(tokenAK), watermark);
+ metadataTableDao.writeNewPartition(newPartitionAK);
assertEquals(
- ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
// Check what parameters were passed to OutputReceiver to verify the new partitions created by
// DNP are correct.
@@ -248,7 +532,7 @@ public class DetectNewPartitionsActionTest {
partitionRecordArgumentCaptor.getAllValues().stream()
.map(PartitionRecord::getPartition)
.collect(Collectors.toList());
- List<ByteStringRange> actualPartitions = Arrays.asList(child1, child2, child3);
+ List<ByteStringRange> actualPartitions = Arrays.asList(childEmptyA, childKEmpty, childAK);
assertThat(expectedPartitions, Matchers.containsInAnyOrder(actualPartitions.toArray()));
for (PartitionRecord value : partitionRecordArgumentCaptor.getAllValues()) {
assertEquals(value.getParentLowWatermark(), watermark);
@@ -264,143 +548,181 @@ public class DetectNewPartitionsActionTest {
value.getChangeStreamContinuationTokens().get(0).getPartition())
.toStringUtf8());
}
+ assertTrue(metadataTableDao.readNewPartitions().isEmpty());
}
@Test
public void testProcessMergeNewPartitions() throws Exception {
- // Avoid 0 and multiples of 10 so that we can specifically test just reading new partitions.
+ // Avoid 0 and multiples of 2 so that we can specifically test just reading new partitions.
OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
// ["a, "b") and ["b", "c") merge into ["a", "c")
- ByteStringRange parentPartition1 = ByteStringRange.create("a", "b");
- Instant watermark1 = startTime;
- ByteStringRange parentPartition2 = ByteStringRange.create("b", "c");
- Instant watermark2 = startTime.plus(Duration.millis(10));
-
- ByteStringRange childPartition = ByteStringRange.create("a", "c");
- ChangeStreamContinuationToken token1 =
- ChangeStreamContinuationToken.create(childPartition, "token1");
- ChangeStreamContinuationToken token2 =
- ChangeStreamContinuationToken.create(childPartition, "token2");
+ ByteStringRange childPartitionAC = ByteStringRange.create("a", "c");
+
+ ByteStringRange parentPartitionAB = ByteStringRange.create("a", "b");
+ Instant watermarkAB = startTime;
+ ChangeStreamContinuationToken tokenAB =
+ ChangeStreamContinuationToken.create(parentPartitionAB, "ab");
+ NewPartition newPartitionACFromAB =
+ new NewPartition(childPartitionAC, Collections.singletonList(tokenAB), watermarkAB);
+ ByteStringRange parentPartitionBC = ByteStringRange.create("b", "c");
+ Instant watermarkBC = startTime.plus(Duration.millis(10));
+ ChangeStreamContinuationToken tokenBC =
+ ChangeStreamContinuationToken.create(parentPartitionBC, "bc");
+ NewPartition newPartitionACFromBC =
+ new NewPartition(childPartitionAC, Collections.singletonList(tokenBC), watermarkBC);
// Write a new partition for every parent partition that merges into the child.
- metadataTableDao.writeNewPartition(childPartition, token1, parentPartition1, watermark1);
- metadataTableDao.writeNewPartition(childPartition, token2, parentPartition2, watermark2);
+ metadataTableDao.writeNewPartition(newPartitionACFromAB);
+ metadataTableDao.writeNewPartition(newPartitionACFromBC);
assertEquals(
- ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
// The partition is outputted with watermark1 because that is the lowest of the 2 forming the
// parent low watermark.
verify(receiver, times(1))
.outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
- assertEquals(childPartition, partitionRecordArgumentCaptor.getValue().getPartition());
- assertEquals(watermark1, partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
+ assertEquals(childPartitionAC, partitionRecordArgumentCaptor.getValue().getPartition());
+ assertEquals(watermarkAB, partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
assertEquals(endTime, partitionRecordArgumentCaptor.getValue().getEndTime());
assertThat(
partitionRecordArgumentCaptor.getValue().getChangeStreamContinuationTokens(),
- Matchers.containsInAnyOrder(token1, token2));
+ Matchers.containsInAnyOrder(tokenAB, tokenBC));
+ assertTrue(metadataTableDao.readNewPartitions().isEmpty());
}
// Test merging partition with a parent partition that hasn't stopped yet.
@Test
public void testProcessMergeNewPartitionsMissingParent() throws Exception {
- // Avoid 0 and multiples of 10 so that we can specifically test just reading new partitions.
+ // Avoid 0 and multiples of 2 so that we can specifically test just reading new partitions.
OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
// ["a, "b") and ["b", "c") merge into ["a", "c") but ["b", "c") is still processing.
- ByteStringRange parentPartition1 = ByteStringRange.create("a", "b");
- Instant watermark1 = startTime;
+ ByteStringRange parentPartitionAB = ByteStringRange.create("a", "b");
+ Instant watermarkAB = startTime;
+ ChangeStreamContinuationToken tokenAB =
+ ChangeStreamContinuationToken.create(parentPartitionAB, "ab");
+
+ ByteStringRange childPartitionAC = ByteStringRange.create("a", "c");
+
+ NewPartition newPartitionACFromAB =
+ new NewPartition(childPartitionAC, Collections.singletonList(tokenAB), watermarkAB);
- ByteStringRange childPartition = ByteStringRange.create("a", "c");
- ChangeStreamContinuationToken token1 =
- ChangeStreamContinuationToken.create(childPartition, "token1");
// Write a new partition for every parent partition that merges into the child.
- metadataTableDao.writeNewPartition(childPartition, token1, parentPartition1, watermark1);
+ metadataTableDao.writeNewPartition(newPartitionACFromAB);
assertEquals(
- ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
- // The partition is outputted with watermark1 because that is the lowest of the 2 forming the
- // parent low watermark.
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+ // No partitions are outputted because it's missing a parent still.
verify(receiver, never()).outputWithTimestamp(any(), any());
+ assertEquals(1, metadataTableDao.readNewPartitions().size());
+
// On next iteration, ["b", "c") received CloseStream and writes to the metadata table.
- ByteStringRange parentPartition2 = ByteStringRange.create("b", "c");
- Instant watermark2 = startTime.plus(Duration.millis(10));
- ChangeStreamContinuationToken token2 =
- ChangeStreamContinuationToken.create(childPartition, "token2");
- metadataTableDao.writeNewPartition(childPartition, token2, parentPartition2, watermark2);
+ ByteStringRange parentPartitionBC = ByteStringRange.create("b", "c");
+ Instant watermarkBC = startTime.plus(Duration.millis(10));
+ ChangeStreamContinuationToken tokenBC =
+ ChangeStreamContinuationToken.create(parentPartitionBC, "bc");
+ NewPartition newPartitionACFromBC =
+ new NewPartition(childPartitionAC, Collections.singletonList(tokenBC), watermarkBC);
+ metadataTableDao.writeNewPartition(newPartitionACFromBC);
assertEquals(
- ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
// The partition is outputted with watermark1 because that is the lowest of the 2 forming the
// parent low watermark.
verify(receiver, times(1))
.outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
- assertEquals(childPartition, partitionRecordArgumentCaptor.getValue().getPartition());
- assertEquals(watermark1, partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
+ assertEquals(childPartitionAC, partitionRecordArgumentCaptor.getValue().getPartition());
+ assertEquals(watermarkAB, partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
assertEquals(endTime, partitionRecordArgumentCaptor.getValue().getEndTime());
assertThat(
partitionRecordArgumentCaptor.getValue().getChangeStreamContinuationTokens(),
- Matchers.containsInAnyOrder(token1, token2));
+ Matchers.containsInAnyOrder(tokenAB, tokenBC));
+ assertTrue(metadataTableDao.readNewPartitions().isEmpty());
}
@Test
public void testMissingPartitionReconciled() throws Exception {
- // We advance watermark on every 10 restriction tracker advancement
- OffsetRange offsetRange = new OffsetRange(10, Long.MAX_VALUE);
+ // We only start reconciling after 50.
+ // We advance watermark on every 2 restriction tracker advancement
+ OffsetRange offsetRange = new OffsetRange(52, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
// Write 2 partitions to the table, missing [a, b)
- ByteStringRange partition1 = ByteStringRange.create("", "a");
- Instant watermark1 = partitionTime.plus(Duration.millis(100));
- metadataTableDao.updateWatermark(partition1, watermark1, null);
- ByteStringRange partition2 = ByteStringRange.create("b", "");
- Instant watermark2 = endTime.plus(Duration.millis(1));
- metadataTableDao.updateWatermark(partition2, watermark2, null);
-
- HashMap<ByteStringRange, Long> missingPartitionDurations = new HashMap<>();
+ ByteStringRange partitionEmptyA = ByteStringRange.create("", "a");
+ Instant watermarkEmptyA = endTime.plus(Duration.millis(100));
+ PartitionRecord partitionRecordEmptyA =
+ new PartitionRecord(
+ partitionEmptyA,
+ watermarkEmptyA,
+ UniqueIdGenerator.getNextId(),
+ watermarkEmptyA,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecordEmptyA);
+ ByteStringRange partitionBEmpty = ByteStringRange.create("b", "");
+ Instant watermarkBEmpty = endTime.plus(Duration.millis(1));
+ PartitionRecord partitionRecordBEmpty =
+ new PartitionRecord(
+ partitionBEmpty,
+ watermarkBEmpty,
+ UniqueIdGenerator.getNextId(),
+ watermarkBEmpty,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecordBEmpty);
+
+ HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
ByteStringRange partitionAB = ByteStringRange.create("a", "b");
// Partition missing for 5 minutes less 1 seconds.
- missingPartitionDurations.put(partitionAB, Instant.now().getMillis() - (5 * 60 - 1) * 1000L);
+ missingPartitionDurations.put(
+ partitionAB, Instant.now().minus(Duration.standardSeconds(10 * 60 - 1)));
metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
// No new partitions and missing partition has not been missing for long enough.
assertEquals(
- ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
verify(receiver, never()).outputWithTimestamp(any(), any());
// Sleep for 1 second, enough that the missing partition needs to be reconciled.
- Thread.sleep(1000);
+ Thread.sleep(1001);
- // We advance the restriction tracker by 1. Because it is not a multiple of 10, we don't
+ // We advance the restriction tracker by 1. Because it is not a multiple of 2, we don't
// evaluate missing partitions, which means we don't perform reconciliation.
- offsetRange = new OffsetRange(11, Long.MAX_VALUE);
+ offsetRange = new OffsetRange(53, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
assertEquals(
- ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
verify(receiver, never()).outputWithTimestamp(any(), any());
- // Multiple of 10, reconciliation should happen.
- offsetRange = new OffsetRange(20, Long.MAX_VALUE);
+ // Multiple of 2, reconciliation should happen.
+ offsetRange = new OffsetRange(54, Long.MAX_VALUE);
when(tracker.currentRestriction()).thenReturn(offsetRange);
when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
assertEquals(
- ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
- action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+ DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+ action.run(
+ tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
verify(receiver, times(1))
.outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
assertEquals(partitionAB, partitionRecordArgumentCaptor.getValue().getPartition());
@@ -409,8 +731,8 @@ public class DetectNewPartitionsActionTest {
partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
assertEquals(endTime, partitionRecordArgumentCaptor.getValue().getEndTime());
assertNotNull(partitionRecordArgumentCaptor.getValue().getStartTime());
- assertEquals(
- watermarkEstimator.currentWatermark(),
- partitionRecordArgumentCaptor.getValue().getStartTime());
+ // The startTime should be the startTime of the pipeline because it's less than 1 hour before
+ // the low watermark.
+ assertEquals(startTime, partitionRecordArgumentCaptor.getValue().getStartTime());
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java
index dcaadd568f8..102200ab9a1 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java
@@ -25,7 +25,7 @@ import static org.mockito.Mockito.when;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
-import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import com.google.cloud.bigtable.data.v2.models.Range;
import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
import java.io.IOException;
import java.util.Arrays;
@@ -35,12 +35,7 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
-import org.apache.beam.sdk.io.range.OffsetRange;
import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
-import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
-import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
-import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Before;
@@ -58,12 +53,10 @@ public class GenerateInitialPartitionsActionTest {
@ClassRule
public static final BigtableEmulatorRule BIGTABLE_EMULATOR_RULE = BigtableEmulatorRule.create();
- @Mock private RestrictionTracker<OffsetRange, Long> tracker;
@Mock private ChangeStreamDao changeStreamDao;
@Mock private ChangeStreamMetrics metrics;
@Mock private OutputReceiver<PartitionRecord> receiver;
- private ManualWatermarkEstimator<Instant> watermarkEstimator;
private Instant startTime;
private Instant endTime;
private static BigtableTableAdminClient adminClient;
@@ -89,23 +82,18 @@ public class GenerateInitialPartitionsActionTest {
metadataTableAdminDao.createMetadataTable();
startTime = Instant.now();
endTime = startTime.plus(Duration.standardSeconds(10));
- watermarkEstimator = new WatermarkEstimators.Manual(startTime);
}
@Test
public void testGenerateInitialPartitionsFromStartTime() {
- when(tracker.tryClaim(0L)).thenReturn(true);
-
- ByteStringRange partition1 = ByteStringRange.create("", "b");
- ByteStringRange partition2 = ByteStringRange.create("b", "");
- List<ByteStringRange> partitionRecordList = Arrays.asList(partition1, partition2);
+ Range.ByteStringRange partition1 = Range.ByteStringRange.create("", "b");
+ Range.ByteStringRange partition2 = Range.ByteStringRange.create("b", "");
+ List<Range.ByteStringRange> partitionRecordList = Arrays.asList(partition1, partition2);
when(changeStreamDao.generateInitialChangeStreamPartitions()).thenReturn(partitionRecordList);
GenerateInitialPartitionsAction generateInitialPartitionsAction =
new GenerateInitialPartitionsAction(metrics, changeStreamDao, endTime);
- assertEquals(
- ProcessContinuation.resume(),
- generateInitialPartitionsAction.run(receiver, tracker, watermarkEstimator, startTime));
+ generateInitialPartitionsAction.run(receiver, startTime);
verify(receiver, times(2))
.outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
List<PartitionRecord> actualPartitions = partitionRecordArgumentCaptor.getAllValues();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java
index 8ff9368cf47..f997db07a8a 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java
@@ -45,6 +45,7 @@ import java.util.Optional;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.ReadChangeStreamPartitionProgressTracker;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress;
@@ -94,13 +95,17 @@ public class ReadChangeStreamPartitionActionTest {
receiver = mock(DoFn.OutputReceiver.class);
watermarkEstimator = mock(ManualWatermarkEstimator.class);
- partition = ByteStringRange.create("A", "B");
+ partition = ByteStringRange.create("A", "C");
uuid = "123456";
Instant startTime = Instant.now();
Instant parentLowWatermark = Instant.now();
- partitionRecord = new PartitionRecord(partition, startTime, uuid, parentLowWatermark, null);
+ partitionRecord =
+ new PartitionRecord(
+ partition, startTime, uuid, parentLowWatermark, Collections.emptyList(), null);
when(tracker.currentRestriction()).thenReturn(restriction);
- when(metadataTableDao.lockPartition(partition, uuid)).thenReturn(true);
+ when(metadataTableDao.lockAndRecordPartition(partitionRecord)).thenReturn(true);
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(true);
+ when(restriction.isEmpty()).thenReturn(true);
when(restriction.getCurrentToken()).thenReturn(null);
when(restriction.getCloseStream()).thenReturn(null);
// Setting watermark estimator to now so we don't debug.
@@ -126,14 +131,63 @@ public class ReadChangeStreamPartitionActionTest {
action.run(partitionRecord, tracker, receiver, watermarkEstimator);
assertEquals(DoFn.ProcessContinuation.stop(), result);
- // Verify that on successful lock, we don't tryClaim on the tracker because we have no outputs.
+ // Verify that on successful lock, we don't tryClaim on the tracker.
verify(tracker, never()).tryClaim(any());
verify(changeStreamAction).run(any(), any(), any(), any(), any(), anyBoolean());
}
@Test
public void testLockingRowFailsStops() throws IOException {
- when(metadataTableDao.lockPartition(partition, uuid)).thenReturn(false);
+ when(metadataTableDao.lockAndRecordPartition(partitionRecord)).thenReturn(false);
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
+
+ final DoFn.ProcessContinuation result =
+ action.run(partitionRecord, tracker, receiver, watermarkEstimator);
+
+ assertEquals(DoFn.ProcessContinuation.stop(), result);
+
+ // On failure to lock, we try to claim a fail to lock, so it will terminate gracefully.
+ StreamProgress streamProgress = new StreamProgress();
+ streamProgress.setFailToLock(true);
+ verify(tracker).tryClaim(streamProgress);
+ verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+ }
+
+ @Test
+ public void testLockingRowNotNeededAfterFirstRun() throws IOException {
+ when(metadataTableDao.lockAndRecordPartition(partitionRecord)).thenReturn(false);
+ // After the first run, we don't try to lock but we check if the lock owner is the same.
+ when(restriction.isEmpty()).thenReturn(false);
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(true);
+
+ final ServerStream<ChangeStreamRecord> responses = mock(ServerStream.class);
+ final Iterator<ChangeStreamRecord> responseIterator = mock(Iterator.class);
+ when(responses.iterator()).thenReturn(responseIterator);
+
+ Heartbeat mockHeartBeat = Mockito.mock(Heartbeat.class);
+ when(responseIterator.next()).thenReturn(mockHeartBeat);
+ when(responseIterator.hasNext()).thenReturn(true);
+ when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any(), anyBoolean()))
+ .thenReturn(responses);
+
+ when(changeStreamAction.run(any(), any(), any(), any(), any(), anyBoolean()))
+ .thenReturn(Optional.of(DoFn.ProcessContinuation.stop()));
+
+ final DoFn.ProcessContinuation result =
+ action.run(partitionRecord, tracker, receiver, watermarkEstimator);
+
+ assertEquals(DoFn.ProcessContinuation.stop(), result);
+ // Verify that on successful lock, we don't tryClaim on the tracker.
+ verify(tracker, never()).tryClaim(any());
+ verify(changeStreamAction).run(any(), any(), any(), any(), any(), anyBoolean());
+ }
+
+ @Test
+ public void testLockingRowNotNeededAfterFirstRunNotSame() throws IOException {
+ when(metadataTableDao.lockAndRecordPartition(partitionRecord)).thenReturn(false);
+ // After the first run, we check if the lock is the same, but in this case it's not.
+ when(restriction.isEmpty()).thenReturn(false);
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
final DoFn.ProcessContinuation result =
action.run(partitionRecord, tracker, receiver, watermarkEstimator);
@@ -174,147 +228,195 @@ public class ReadChangeStreamPartitionActionTest {
@Test
public void testCloseStreamTerminateOKStatus() throws IOException {
+ // Force lock fail because CloseStream should not depend on locking
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
Status statusProto = Status.newBuilder().setCode(0).build();
Mockito.when(mockCloseStream.getStatus())
.thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+ when(restriction.isEmpty()).thenReturn(false);
final DoFn.ProcessContinuation result =
action.run(partitionRecord, tracker, receiver, watermarkEstimator);
assertEquals(DoFn.ProcessContinuation.stop(), result);
// Should terminate before reaching processing stream partition responses.
verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+ // Should not try claim any restriction when processing CloseStream
+ verify(tracker, (never())).tryClaim(any());
// Should decrement the metric on termination.
verify(metrics).decPartitionStreamCount();
// Should not try to write any new partition to the metadata table.
- verify(metadataTableDao, never()).writeNewPartition(any(), any(), any(), any());
+ verify(metadataTableDao, never()).writeNewPartition(any());
+ verify(metadataTableDao, never()).releaseStreamPartitionLockForDeletion(any(), any());
verify(metadataTableDao, never()).deleteStreamPartitionRow(any());
}
@Test
public void testCloseStreamTerminateNotOutOfRangeStatus() throws IOException {
+ // Force lock fail because CloseStream should not depend on locking
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
// Out of Range code is 11.
CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
Status statusProto = Status.newBuilder().setCode(10).build();
Mockito.when(mockCloseStream.getStatus())
.thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+ when(restriction.isEmpty()).thenReturn(false);
final DoFn.ProcessContinuation result =
action.run(partitionRecord, tracker, receiver, watermarkEstimator);
assertEquals(DoFn.ProcessContinuation.stop(), result);
// Should terminate before reaching processing stream partition responses.
verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+ // Should not try claim any restriction when processing CloseStream
+ verify(tracker, (never())).tryClaim(any());
// Should decrement the metric on termination.
verify(metrics).decPartitionStreamCount();
// Should not try to write any new partition to the metadata table.
- verify(metadataTableDao, never()).writeNewPartition(any(), any(), any(), any());
+ verify(metadataTableDao, never()).writeNewPartition(any());
+ verify(metadataTableDao, never()).releaseStreamPartitionLockForDeletion(any(), any());
verify(metadataTableDao, never()).deleteStreamPartitionRow(any());
}
@Test
public void testCloseStreamWritesContinuationTokens() throws IOException {
- ChangeStreamContinuationToken changeStreamContinuationToken1 =
- ChangeStreamContinuationToken.create(ByteStringRange.create("A", "AJ"), "1234");
- ChangeStreamContinuationToken changeStreamContinuationToken2 =
- ChangeStreamContinuationToken.create(ByteStringRange.create("AJ", "B"), "5678");
+ // Force lock fail because CloseStream should not depend on locking
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
+ ChangeStreamContinuationToken tokenAB =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+ ChangeStreamContinuationToken tokenBC =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
Status statusProto = Status.newBuilder().setCode(11).build();
Mockito.when(mockCloseStream.getStatus())
.thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
Mockito.when(mockCloseStream.getChangeStreamContinuationTokens())
- .thenReturn(Arrays.asList(changeStreamContinuationToken1, changeStreamContinuationToken2));
+ .thenReturn(Arrays.asList(tokenAB, tokenBC));
Mockito.when(mockCloseStream.getNewPartitions())
- .thenReturn(
- Arrays.asList(
- changeStreamContinuationToken1.getPartition(),
- changeStreamContinuationToken2.getPartition()));
+ .thenReturn(Arrays.asList(tokenAB.getPartition(), tokenBC.getPartition()));
when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+ when(restriction.isEmpty()).thenReturn(false);
final DoFn.ProcessContinuation result =
action.run(partitionRecord, tracker, receiver, watermarkEstimator);
assertEquals(DoFn.ProcessContinuation.stop(), result);
// Should terminate before reaching processing stream partition responses.
verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+ // Should not try claim any restriction when processing CloseStream
+ verify(tracker, (never())).tryClaim(any());
// Should decrement the metric on termination.
verify(metrics).decPartitionStreamCount();
// Write the new partitions.
- verify(metadataTableDao)
- .writeNewPartition(any(), eq(changeStreamContinuationToken1), any(), any());
- verify(metadataTableDao)
- .writeNewPartition(any(), eq(changeStreamContinuationToken2), any(), any());
+ NewPartition newPartitionAB =
+ new NewPartition(
+ tokenAB.getPartition(),
+ Collections.singletonList(tokenAB),
+ watermarkEstimator.getState());
+ verify(metadataTableDao).writeNewPartition(newPartitionAB);
+ NewPartition newPartitionBC =
+ new NewPartition(
+ tokenBC.getPartition(),
+ Collections.singletonList(tokenBC),
+ watermarkEstimator.getState());
+ verify(metadataTableDao).writeNewPartition(newPartitionBC);
+ verify(metadataTableDao, times(1))
+ .releaseStreamPartitionLockForDeletion(
+ partitionRecord.getPartition(), partitionRecord.getUuid());
verify(metadataTableDao, times(1)).deleteStreamPartitionRow(partitionRecord.getPartition());
}
@Test
public void testCloseStreamNewPartitionMerge() throws IOException {
+ // Force lock fail because CloseStream should not depend on locking
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
// NewPartitions field includes the merge target. ChangeStreamContinuationToken's partition may
// not be the same as the new partition.
- // AB merging into AC. The new partition is AC and the corresponding
- // ChangeStreamContinuationToken is for AB
- ByteStringRange childPartitionAC = ByteStringRange.create("A", "C");
- ChangeStreamContinuationToken parentTokenAB =
- ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+ // AC merging into AD. The new partition is AD and the corresponding
+ // ChangeStreamContinuationToken is for AC
+ ByteStringRange childPartitionAD = ByteStringRange.create("A", "D");
+ ChangeStreamContinuationToken parentTokenAC =
+ ChangeStreamContinuationToken.create(partition, "AC");
CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
Status statusProto = Status.newBuilder().setCode(11).build();
Mockito.when(mockCloseStream.getStatus())
.thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
Mockito.when(mockCloseStream.getChangeStreamContinuationTokens())
- .thenReturn(Collections.singletonList(parentTokenAB));
+ .thenReturn(Collections.singletonList(parentTokenAC));
Mockito.when(mockCloseStream.getNewPartitions())
- .thenReturn(Collections.singletonList(childPartitionAC));
+ .thenReturn(Collections.singletonList(childPartitionAD));
+ when(restriction.isEmpty()).thenReturn(false);
when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+ when(restriction.isEmpty()).thenReturn(false);
final DoFn.ProcessContinuation result =
action.run(partitionRecord, tracker, receiver, watermarkEstimator);
assertEquals(DoFn.ProcessContinuation.stop(), result);
// Should terminate before reaching processing stream partition responses.
verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+ // Should not try claim any restriction when processing CloseStream
+ verify(tracker, (never())).tryClaim(any());
// Should decrement the metric on termination.
verify(metrics).decPartitionStreamCount();
// Write the new partitions.
- // Write the new partitions.
- verify(metadataTableDao)
- .writeNewPartition(eq(childPartitionAC), eq(parentTokenAB), eq(partition), any());
- verify(metadataTableDao, times(1)).deleteStreamPartitionRow(partitionRecord.getPartition());
+ NewPartition newPartitionAD =
+ new NewPartition(
+ childPartitionAD,
+ Collections.singletonList(parentTokenAC),
+ watermarkEstimator.getState());
+ verify(metadataTableDao).writeNewPartition(eq(newPartitionAD));
+ verify(metadataTableDao, times(1))
+ .releaseStreamPartitionLockForDeletion(
+ partitionRecord.getPartition(), partitionRecord.getUuid());
verify(metadataTableDao, times(1)).deleteStreamPartitionRow(partitionRecord.getPartition());
}
@Test
public void testCloseStreamMergeWithoutNewPartitionsField() throws IOException {
+ // Force lock fail because CloseStream should not depend on locking
+ when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
// This is testing the old version of the API without the NewPartitions field. This means the
// ChangeStreamContinuationToken's partition represents the new partition.
- // AB merging into AC.
- ByteStringRange childPartitionAC = ByteStringRange.create("A", "C");
+ // AC merging into AD.
+ ByteStringRange childPartitionAD = ByteStringRange.create("A", "D");
// The partition in the token is different from the test above. The token is actually encoded
- // for partition AB, but in this version, the partition in the token represents the NEW (child)
+ // for partition AC, but in this version, the partition in the token represents the NEW (child)
// partition. This has been replaced by the new_partitions field in CloseStream.
- ChangeStreamContinuationToken parentTokenAB =
- ChangeStreamContinuationToken.create(ByteStringRange.create("A", "C"), "AB");
+ ChangeStreamContinuationToken parentTokenAC =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("A", "D"), "AC");
CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
Status statusProto = Status.newBuilder().setCode(11).build();
Mockito.when(mockCloseStream.getStatus())
.thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
Mockito.when(mockCloseStream.getChangeStreamContinuationTokens())
- .thenReturn(Collections.singletonList(parentTokenAB));
+ .thenReturn(Collections.singletonList(parentTokenAC));
Mockito.when(mockCloseStream.getNewPartitions()).thenReturn(Collections.emptyList());
when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+ when(restriction.isEmpty()).thenReturn(false);
final DoFn.ProcessContinuation result =
action.run(partitionRecord, tracker, receiver, watermarkEstimator);
assertEquals(DoFn.ProcessContinuation.stop(), result);
// Should terminate before reaching processing stream partition responses.
verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+ // Should not try claim any restriction when processing CloseStream
+ verify(tracker, (never())).tryClaim(any());
// Should decrement the metric on termination.
verify(metrics).decPartitionStreamCount();
// We have to correct the partition in the tokens if we don't have new_partitions field.
- ChangeStreamContinuationToken realTokenAB =
- getTokenWithCorrectPartition(ByteStringRange.create("A", "B"), parentTokenAB);
+ ChangeStreamContinuationToken realTokenAC =
+ getTokenWithCorrectPartition(partition, parentTokenAC);
// Write the new partitions.
- verify(metadataTableDao)
- .writeNewPartition(eq(childPartitionAC), eq(realTokenAB), eq(partition), any());
+ NewPartition newPartitionAD =
+ new NewPartition(
+ childPartitionAD,
+ Collections.singletonList(realTokenAC),
+ watermarkEstimator.getState());
+ verify(metadataTableDao).writeNewPartition(eq(newPartitionAD));
+ verify(metadataTableDao, times(1))
+ .releaseStreamPartitionLockForDeletion(
+ partitionRecord.getPartition(), partitionRecord.getUuid());
verify(metadataTableDao, times(1)).deleteStreamPartitionRow(partitionRecord.getPartition());
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java
new file mode 100644
index 00000000000..3f8737c0595
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
+import com.google.cloud.bigtable.data.v2.BigtableDataClient;
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ResumeFromPreviousPipelineActionTest {
+ @ClassRule
+ public static final BigtableEmulatorRule BIGTABLE_EMULATOR_RULE = BigtableEmulatorRule.create();
+
+ private MetadataTableDao metadataTableDao;
+ @Mock private ChangeStreamMetrics metrics;
+ @Mock private OutputReceiver<PartitionRecord> receiver;
+
+ private Instant endTime;
+ private static BigtableDataClient dataClient;
+ private static BigtableTableAdminClient adminClient;
+
+ private ResumeFromPreviousPipelineAction action;
+
+ @Captor ArgumentCaptor<PartitionRecord> partitionRecordArgumentCaptor;
+
+ @BeforeClass
+ public static void beforeClass() throws IOException {
+ BigtableTableAdminSettings adminSettings =
+ BigtableTableAdminSettings.newBuilderForEmulator(BIGTABLE_EMULATOR_RULE.getPort())
+ .setProjectId("fake-project")
+ .setInstanceId("fake-instance")
+ .build();
+ adminClient = BigtableTableAdminClient.create(adminSettings);
+ BigtableDataSettings dataSettingsBuilder =
+ BigtableDataSettings.newBuilderForEmulator(BIGTABLE_EMULATOR_RULE.getPort())
+ .setProjectId("fake-project")
+ .setInstanceId("fake-instance")
+ .build();
+ dataClient = BigtableDataClient.create(dataSettingsBuilder);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ String changeStreamId = UniqueIdGenerator.generateRowKeyPrefix();
+ MetadataTableAdminDao metadataTableAdminDao =
+ new MetadataTableAdminDao(
+ adminClient, null, changeStreamId, MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME);
+ metadataTableAdminDao.createMetadataTable();
+ metadataTableDao =
+ new MetadataTableDao(
+ dataClient,
+ metadataTableAdminDao.getTableId(),
+ metadataTableAdminDao.getChangeStreamNamePrefix());
+ endTime = Instant.now().plus(Duration.standardSeconds(10));
+ ProcessNewPartitionsAction processNewPartitionsAction =
+ new ProcessNewPartitionsAction(metrics, metadataTableDao, endTime);
+ action =
+ new ResumeFromPreviousPipelineAction(
+ metrics, metadataTableDao, endTime, processNewPartitionsAction);
+ }
+
+ @Test
+ public void testResetMissingPartitions() throws InvalidProtocolBufferException {
+ HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
+ missingPartitionsDuration.put(ByteStringRange.create("A", "B"), Instant.now());
+ metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionsDuration);
+ assertFalse(metadataTableDao.readDetectNewPartitionMissingPartitions().isEmpty());
+ action.run(receiver);
+ assertTrue(metadataTableDao.readDetectNewPartitionMissingPartitions().isEmpty());
+ }
+
+ @Test
+ public void testOutputExistingStreamPartitions() throws InvalidProtocolBufferException {
+ ByteStringRange partition1 = ByteStringRange.create("A", "B");
+ Instant watermark1 = Instant.now().minus(Duration.standardSeconds(10));
+ PartitionRecord partitionWithStartTime =
+ new PartitionRecord(partition1, watermark1, "1", watermark1, Collections.emptyList(), null);
+ metadataTableDao.lockAndRecordPartition(partitionWithStartTime);
+
+ ByteStringRange partition2 = ByteStringRange.create("B", "D");
+ ChangeStreamContinuationToken partition2Token1 =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "tokenBC");
+ ChangeStreamContinuationToken partition2Token2 =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("C", "D"), "tokenCD");
+ Instant watermark2 = Instant.now().plus(Duration.standardMinutes(1));
+ PartitionRecord partitionWithInitialTokens =
+ new PartitionRecord(
+ partition2,
+ Arrays.asList(partition2Token1, partition2Token2),
+ "2",
+ watermark2,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionWithInitialTokens);
+
+ ByteStringRange partition3 = ByteStringRange.create("D", "H");
+ Instant watermark3 = Instant.now();
+ PartitionRecord partitionWithContinuationToken =
+ new PartitionRecord(partition3, watermark3, "3", watermark3, Collections.emptyList(), null);
+ metadataTableDao.lockAndRecordPartition(partitionWithContinuationToken);
+ ChangeStreamContinuationToken partition3token =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("D", "H"), "tokenDH");
+ metadataTableDao.updateWatermark(partition3, watermark3, partition3token);
+
+ ByteStringRange partition4 = ByteStringRange.create("H", "I");
+ ChangeStreamContinuationToken partition4token =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("H", "I"), "tokenHI");
+ Instant watermark4 = Instant.now();
+ PartitionRecord unlockedPartition =
+ new PartitionRecord(
+ partition4,
+ Collections.singletonList(partition4token),
+ "4",
+ watermark4,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(unlockedPartition);
+ metadataTableDao.releaseStreamPartitionLockForDeletion(partition4, unlockedPartition.getUuid());
+
+ action.run(receiver);
+ verify(receiver, times(4))
+ .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
+ List<PartitionRecord> actualPartitions = partitionRecordArgumentCaptor.getAllValues();
+
+ PartitionRecord actualPartition1 = actualPartitions.get(0);
+ assertEquals(partition1, actualPartition1.getPartition());
+ assertEquals(partitionWithStartTime.getUuid(), actualPartition1.getUuid());
+ assertNull(actualPartition1.getChangeStreamContinuationTokens());
+ assertEquals(partitionWithStartTime.getStartTime(), actualPartition1.getStartTime());
+ assertEquals(endTime, actualPartition1.getEndTime());
+ assertEquals(
+ partitionWithStartTime.getParentLowWatermark(), actualPartition1.getParentLowWatermark());
+
+ PartitionRecord actualPartition2 = actualPartitions.get(1);
+ assertEquals(partition2, actualPartition2.getPartition());
+ assertEquals(partitionWithInitialTokens.getUuid(), actualPartition2.getUuid());
+ assertNull(actualPartition2.getStartTime());
+ assertEquals(
+ partitionWithInitialTokens.getChangeStreamContinuationTokens(),
+ actualPartition2.getChangeStreamContinuationTokens());
+ assertEquals(endTime, actualPartition2.getEndTime());
+ assertEquals(
+ partitionWithInitialTokens.getParentLowWatermark(),
+ actualPartition2.getParentLowWatermark());
+
+ PartitionRecord actualPartition3 = actualPartitions.get(2);
+ assertEquals(partition3, actualPartition3.getPartition());
+ assertEquals(partitionWithContinuationToken.getUuid(), actualPartition3.getUuid());
+ assertNull(actualPartition3.getStartTime());
+ assertNotNull(actualPartition3.getChangeStreamContinuationTokens());
+ assertEquals(1, actualPartition3.getChangeStreamContinuationTokens().size());
+ assertEquals(partition3token, actualPartition3.getChangeStreamContinuationTokens().get(0));
+ assertEquals(endTime, actualPartition3.getEndTime());
+ assertEquals(
+ partitionWithContinuationToken.getParentLowWatermark(),
+ actualPartition3.getParentLowWatermark());
+
+ PartitionRecord actualPartition4 = actualPartitions.get(3);
+ assertEquals(partition4, actualPartition4.getPartition());
+ // partition4 was unlocked so it gets a new uuid.
+ assertNotEquals("4", actualPartition4.getUuid());
+ assertNull(actualPartition4.getStartTime());
+ assertNotNull(actualPartition4.getChangeStreamContinuationTokens());
+ assertEquals(1, actualPartition4.getChangeStreamContinuationTokens().size());
+ assertEquals(partition4token, actualPartition4.getChangeStreamContinuationTokens().get(0));
+ assertEquals(endTime, actualPartition4.getEndTime());
+ assertEquals(
+ unlockedPartition.getParentLowWatermark(), actualPartition4.getParentLowWatermark());
+ }
+
+ @Test
+ public void testOutputNewPartitions() throws InvalidProtocolBufferException {
+ ByteStringRange partition1 = ByteStringRange.create("A", "B");
+ ChangeStreamContinuationToken token1 =
+ ChangeStreamContinuationToken.create(partition1, "tokenAB");
+ Instant watermark1 = Instant.now().minus(Duration.standardSeconds(10));
+ NewPartition newSplitPartition =
+ new NewPartition(partition1, Collections.singletonList(token1), watermark1);
+ metadataTableDao.writeNewPartition(newSplitPartition);
+
+ ByteStringRange partition2 = ByteStringRange.create("B", "D");
+ ByteStringRange partition2parent1 = ByteStringRange.create("C", "D");
+ ChangeStreamContinuationToken token2 =
+ ChangeStreamContinuationToken.create(partition2parent1, "tokenCD");
+ Instant watermark2 = Instant.now().plus(Duration.standardSeconds(10));
+ NewPartition newPartitionMissingParent =
+ new NewPartition(partition2, Collections.singletonList(token2), watermark2);
+ metadataTableDao.writeNewPartition(newPartitionMissingParent);
+
+ ByteStringRange partition3 = ByteStringRange.create("D", "E");
+ ChangeStreamContinuationToken token3 =
+ ChangeStreamContinuationToken.create(partition3, "tokenDE");
+ Instant watermark3 = Instant.now().plus(Duration.standardSeconds(5));
+ NewPartition deletedNewPartition =
+ new NewPartition(partition3, Collections.singletonList(token3), watermark3);
+ metadataTableDao.writeNewPartition(deletedNewPartition);
+ metadataTableDao.markNewPartitionForDeletion(deletedNewPartition);
+
+ // There are only 2 NewPartition rows because partition3 is deleted.
+ assertEquals(2, metadataTableDao.readNewPartitions().size());
+
+ action.run(receiver);
+ verify(receiver, times(2))
+ .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
+ List<PartitionRecord> actualPartitions = partitionRecordArgumentCaptor.getAllValues();
+
+ PartitionRecord actualPartition1 = actualPartitions.get(0);
+ assertEquals(partition1, actualPartition1.getPartition());
+ assertEquals(1, actualPartition1.getChangeStreamContinuationTokens().size());
+ assertEquals(token1, actualPartition1.getChangeStreamContinuationTokens().get(0));
+ assertNull(actualPartition1.getStartTime());
+ assertEquals(watermark1, actualPartition1.getParentLowWatermark());
+ assertEquals(endTime, actualPartition1.getEndTime());
+ assertEquals(
+ Collections.singletonList(newSplitPartition), actualPartition1.getParentPartitions());
+ // Uuid is filled.
+ assertFalse(actualPartition1.getUuid().isEmpty());
+
+ // The 2nd partition is not partition2 but partition3 because partition2 is missing a parent.
+ // Even though partition3 is marked for deletion, we still process it.
+ PartitionRecord actualPartition2 = actualPartitions.get(1);
+ assertEquals(partition3, actualPartition2.getPartition());
+ assertEquals(1, actualPartition2.getChangeStreamContinuationTokens().size());
+ assertEquals(token3, actualPartition2.getChangeStreamContinuationTokens().get(0));
+ assertNull(actualPartition2.getStartTime());
+ assertEquals(watermark3, actualPartition2.getParentLowWatermark());
+ assertEquals(endTime, actualPartition2.getEndTime());
+ assertEquals(
+ Collections.singletonList(deletedNewPartition), actualPartition2.getParentPartitions());
+
+ // There is only 1 NewPartition row now because we processed them except for the NewPartition
+ // with missing parent.
+ assertEquals(1, metadataTableDao.readNewPartitions().size());
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java
index c7f915da638..a98814d4f48 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java
@@ -17,11 +17,15 @@
*/
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseWatermarkFromRow;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import com.google.api.gax.rpc.ServerStream;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
@@ -36,10 +40,16 @@ import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.StreamPartitionWithWatermark;
+import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -52,6 +62,7 @@ import org.slf4j.LoggerFactory;
@RunWith(JUnit4.class)
public class MetadataTableDaoTest {
+
private static final Logger LOG = LoggerFactory.getLogger(MetadataTableDaoTest.class);
@ClassRule
@@ -135,11 +146,13 @@ public class MetadataTableDaoTest {
ByteString rowKey = metadataTableDao.convertPartitionToStreamPartitionRowKey(partition);
// Class to try to lock the partition in a separate thread.
class LockPartition implements Runnable {
- final String id;
+ final PartitionRecord partitionRecord =
+ new PartitionRecord(
+ partition, Collections.emptyList(), Instant.now(), Collections.emptyList());
boolean locked = false;
LockPartition(String id) {
- this.id = id;
+ partitionRecord.setUuid(id);
}
@Override
@@ -149,7 +162,7 @@ public class MetadataTableDaoTest {
// race.
int sleep = (int) (Math.random() * 1000);
Thread.sleep(sleep);
- if (metadataTableDao.lockPartition(partition, id)) {
+ if (metadataTableDao.lockAndRecordPartition(partitionRecord)) {
locked = true;
}
} catch (InterruptedException e) {
@@ -212,58 +225,65 @@ public class MetadataTableDaoTest {
@Test
public void testReadStreamPartitionsWithWatermark() throws InvalidProtocolBufferException {
- ByteStringRange partitionWithWatermark = ByteStringRange.create("a", "");
+ ByteStringRange lockedPartition = ByteStringRange.create("", "a");
+ PartitionRecord partitionRecord =
+ new PartitionRecord(
+ lockedPartition,
+ Instant.now(),
+ UniqueIdGenerator.getNextId(),
+ Instant.now(),
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord);
+
+ // Only one row has both watermark and is locked.
+ List<StreamPartitionWithWatermark> streamPartitionsWithWatermark =
+ metadataTableDao.readStreamPartitionsWithWatermark();
+ assertEquals(1, streamPartitionsWithWatermark.size());
+ assertEquals(
+ partitionRecord.getParentLowWatermark(),
+ streamPartitionsWithWatermark.get(0).getWatermark());
+ assertEquals(
+ partitionRecord.getPartition(), streamPartitionsWithWatermark.get(0).getPartition());
+
Instant watermark = Instant.now();
- metadataTableDao.updateWatermark(partitionWithWatermark, watermark, null);
-
- // This should only return rows where the watermark has been set, not rows that have been locked
- // but have not yet set the first watermark
- ServerStream<Row> rowsWithWatermark =
- metadataTableDao.readFromMdTableStreamPartitionsWithWatermark();
- ArrayList<Row> metadataRows = new ArrayList<>();
- for (Row row : rowsWithWatermark) {
- metadataRows.add(row);
- }
- assertEquals(1, metadataRows.size());
- Instant metadataWatermark = MetadataTableEncoder.parseWatermarkFromRow(metadataRows.get(0));
- assertEquals(watermark, metadataWatermark);
- ByteStringRange rowKeyResponse =
- metadataTableDao.convertStreamPartitionRowKeyToPartition(metadataRows.get(0).getKey());
- assertEquals(partitionWithWatermark, rowKeyResponse);
+ // Update the watermark on the locked partition.
+ metadataTableDao.updateWatermark(lockedPartition, watermark, null);
+
+ // Watermark is updated.
+ streamPartitionsWithWatermark = metadataTableDao.readStreamPartitionsWithWatermark();
+ assertEquals(1, streamPartitionsWithWatermark.size());
+ assertEquals(watermark, streamPartitionsWithWatermark.get(0).getWatermark());
}
@Test
public void testNewPartitionsWriteRead() throws InvalidProtocolBufferException {
// This test a split of ["", "") to ["", "a") and ["a", "")
- ByteStringRange parentPartition = ByteStringRange.create("", "");
- ByteStringRange partition1 = ByteStringRange.create("", "a");
- ChangeStreamContinuationToken changeStreamContinuationToken1 =
- ChangeStreamContinuationToken.create(partition1, "tk1");
- ByteStringRange partition2 = ByteStringRange.create("a", "");
- ChangeStreamContinuationToken changeStreamContinuationToken2 =
- ChangeStreamContinuationToken.create(partition2, "tk2");
-
Instant lowWatermark = Instant.now();
- metadataTableDao.writeNewPartition(
- partition1, changeStreamContinuationToken1, parentPartition, lowWatermark);
- metadataTableDao.writeNewPartition(
- partition2, changeStreamContinuationToken2, parentPartition, lowWatermark);
+ ByteStringRange partitionEmptyA = ByteStringRange.create("", "a");
+ ChangeStreamContinuationToken tokenEmptyA =
+ ChangeStreamContinuationToken.create(partitionEmptyA, "EmptyA");
+ NewPartition newPartitionEmptyA =
+ new NewPartition(partitionEmptyA, Collections.singletonList(tokenEmptyA), lowWatermark);
- ServerStream<Row> rows = metadataTableDao.readNewPartitions();
+ ByteStringRange partitionAEmpty = ByteStringRange.create("a", "");
+ ChangeStreamContinuationToken tokenAEmpty =
+ ChangeStreamContinuationToken.create(partitionAEmpty, "AEmpty");
+ NewPartition newPartitionAEmpty =
+ new NewPartition(partitionAEmpty, Collections.singletonList(tokenAEmpty), lowWatermark);
+
+ metadataTableDao.writeNewPartition(newPartitionEmptyA);
+ metadataTableDao.writeNewPartition(newPartitionAEmpty);
+
+ List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
int rowsCount = 0;
boolean matchedPartition1 = false;
boolean matchedPartition2 = false;
- for (Row row : rows) {
+ for (NewPartition newPartition : newPartitions) {
rowsCount++;
- ByteString newPartitionPrefix =
- metadataTableDao
- .getChangeStreamNamePrefix()
- .concat(MetadataTableAdminDao.NEW_PARTITION_PREFIX);
- ByteStringRange partition =
- ByteStringRange.toByteStringRange(row.getKey().substring(newPartitionPrefix.size()));
- if (partition.equals(partition1)) {
+ if (newPartition.getPartition().equals(partitionEmptyA)) {
matchedPartition1 = true;
- } else if (partition.equals(partition2)) {
+ } else if (newPartition.getPartition().equals(partitionAEmpty)) {
matchedPartition2 = true;
}
}
@@ -272,34 +292,263 @@ public class MetadataTableDaoTest {
assertEquals(2, rowsCount);
}
+ @Test
+ public void testMarkNewPartitionForDeletion() throws InvalidProtocolBufferException {
+ ByteStringRange childPartition = ByteStringRange.create("A", "C");
+ ChangeStreamContinuationToken token =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
+ NewPartition newPartition =
+ new NewPartition(childPartition, Collections.singletonList(token), Instant.now());
+ metadataTableDao.writeNewPartition(newPartition);
+ List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+ assertEquals(1, newPartitions.size());
+ assertEquals(childPartition, newPartitions.get(0).getPartition());
+
+ // Once it's marked for deletion, it is not visible to readNewPartitions.
+ metadataTableDao.markNewPartitionForDeletion(newPartitions.get(0));
+ assertEquals(0, metadataTableDao.readNewPartitions().size());
+ }
+
+ @Test
+ public void testMarkNewPartitionForDeletionVisibleAfterOneMinute()
+ throws InvalidProtocolBufferException {
+ ByteStringRange childPartition = ByteStringRange.create("A", "C");
+ ByteStringRange parentPartition = ByteStringRange.create("B", "C");
+ ChangeStreamContinuationToken token =
+ ChangeStreamContinuationToken.create(parentPartition, "BC");
+ NewPartition newPartition =
+ new NewPartition(childPartition, Collections.singletonList(token), Instant.now());
+ metadataTableDao.writeNewPartition(newPartition);
+ List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+ assertEquals(1, newPartitions.size());
+ assertEquals(childPartition, newPartitions.get(0).getPartition());
+
+ // Mark it for deletion but more than 60s ago.
+ RowMutation rowMutation =
+ RowMutation.create(
+ metadataTableAdminDao.getTableId(),
+ metadataTableDao.convertPartitionToNewPartitionRowKey(childPartition))
+ .setCell(
+ MetadataTableAdminDao.CF_SHOULD_DELETE,
+ ByteStringRange.serializeToByteString(parentPartition),
+ Instant.now().minus(Duration.standardSeconds(61)).getMillis() * 1_000L,
+ 1);
+ dataClient.mutateRow(rowMutation);
+
+ // We should be able to read back the NewPartition.
+ newPartitions = metadataTableDao.readNewPartitions();
+ assertEquals(1, newPartitions.size());
+ assertEquals(childPartition, newPartitions.get(0).getPartition());
+ }
+
+ @Test
+ public void testDeleteNewPartition() throws InvalidProtocolBufferException {
+ ByteStringRange childPartition = ByteStringRange.create("A", "C");
+ ChangeStreamContinuationToken token =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
+ NewPartition newPartition =
+ new NewPartition(childPartition, Collections.singletonList(token), Instant.now());
+ metadataTableDao.writeNewPartition(newPartition);
+ ChangeStreamContinuationToken token2 =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+ NewPartition newPartition2 =
+ new NewPartition(childPartition, Collections.singletonList(token2), Instant.now());
+ metadataTableDao.writeNewPartition(newPartition2);
+
+ List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+ assertEquals(1, newPartitions.size());
+ assertEquals(childPartition, newPartitions.get(0).getPartition());
+
+ // Did not mark new partition for deletion. Cannot delete.
+ assertFalse(metadataTableDao.deleteNewPartition(newPartitions.get(0)));
+ assertEquals(1, metadataTableDao.readNewPartitions().size());
+
+ metadataTableDao.markNewPartitionForDeletion(newPartitions.get(0));
+ assertTrue(metadataTableDao.deleteNewPartition(newPartitions.get(0)));
+ assertEquals(0, metadataTableDao.readNewPartitions().size());
+ }
+
+ @Test
+ public void testPartialDeleteNewPartition() throws InvalidProtocolBufferException {
+ ByteStringRange childPartition = ByteStringRange.create("A", "C");
+ ChangeStreamContinuationToken token =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
+ NewPartition newPartition =
+ new NewPartition(childPartition, Collections.singletonList(token), Instant.now());
+ metadataTableDao.writeNewPartition(newPartition);
+ ChangeStreamContinuationToken token2 =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+ NewPartition newPartition2 =
+ new NewPartition(childPartition, Collections.singletonList(token2), Instant.now());
+ metadataTableDao.writeNewPartition(newPartition2);
+
+ List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+ assertEquals(1, newPartitions.size());
+ assertEquals(childPartition, newPartitions.get(0).getPartition());
+ assertEquals(2, newPartitions.get(0).getChangeStreamContinuationTokens().size());
+
+ // Only mark one of the parent for deletion but try to delete the entire new partition. This
+ // will succeed but only the parent marked for deletion will be deleted.
+ metadataTableDao.markNewPartitionForDeletion(newPartition);
+ assertFalse(metadataTableDao.deleteNewPartition(newPartitions.get(0)));
+ newPartitions = metadataTableDao.readNewPartitions();
+ assertEquals(1, newPartitions.size());
+ assertEquals(childPartition, newPartitions.get(0).getPartition());
+ assertEquals(1, newPartitions.get(0).getChangeStreamContinuationTokens().size());
+ assertEquals(token2, newPartitions.get(0).getChangeStreamContinuationTokens().get(0));
+
+ // Try deleting the parent that isn't marked for deletion. It is not possible
+ assertFalse(metadataTableDao.deleteNewPartition(newPartition2));
+ newPartitions = metadataTableDao.readNewPartitions();
+ assertEquals(1, newPartitions.size());
+ assertEquals(childPartition, newPartitions.get(0).getPartition());
+ assertEquals(1, newPartitions.get(0).getChangeStreamContinuationTokens().size());
+ assertEquals(token2, newPartitions.get(0).getChangeStreamContinuationTokens().get(0));
+
+ // Mark the remaining parent for deletion.
+ metadataTableDao.markNewPartitionForDeletion(newPartition2);
+ assertTrue(metadataTableDao.deleteNewPartition(newPartition2));
+ assertEquals(0, metadataTableDao.readNewPartitions().size());
+ }
+
@Test
public void testUpdateAndReadWatermark() throws InvalidProtocolBufferException {
ByteStringRange partition1 = ByteStringRange.create("a", "b");
Instant watermark1 = Instant.now();
+
+ PartitionRecord partitionRecord =
+ new PartitionRecord(
+ partition1,
+ Instant.now(),
+ UniqueIdGenerator.getNextId(),
+ Instant.now(),
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord);
metadataTableDao.updateWatermark(partition1, watermark1, null);
ByteStringRange partition2 = ByteStringRange.create("b", "c");
Instant watermark2 = Instant.now();
+ PartitionRecord partitionRecord2 =
+ new PartitionRecord(
+ partition2,
+ Instant.now(),
+ UniqueIdGenerator.getNextId(),
+ Instant.now(),
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord2);
metadataTableDao.updateWatermark(partition2, watermark2, null);
- ServerStream<Row> rows = metadataTableDao.readFromMdTableStreamPartitionsWithWatermark();
- int rowsCount = 0;
- boolean matchedPartition1 = false;
- boolean matchedPartition2 = false;
- for (Row row : rows) {
- rowsCount++;
- ByteStringRange partition =
- metadataTableDao.convertStreamPartitionRowKeyToPartition(row.getKey());
- if (partition.equals(partition1)) {
- assertEquals(watermark1, MetadataTableEncoder.parseWatermarkFromRow(row));
- matchedPartition1 = true;
- } else if (partition.equals(partition2)) {
- assertEquals(watermark2, MetadataTableEncoder.parseWatermarkFromRow(row));
- matchedPartition2 = true;
- }
- }
- assertEquals(2, rowsCount);
- assertTrue(matchedPartition1);
- assertTrue(matchedPartition2);
+ List<StreamPartitionWithWatermark> streamPartitionsWithWatermark =
+ metadataTableDao.readStreamPartitionsWithWatermark();
+ assertThat(
+ streamPartitionsWithWatermark,
+ containsInAnyOrder(
+ new StreamPartitionWithWatermark(partition1, watermark1),
+ new StreamPartitionWithWatermark(partition2, watermark2)));
+ }
+
+ @Test
+ public void testReadingExistingStreamPartitions() throws InvalidProtocolBufferException {
+ ByteStringRange partition1 = ByteStringRange.create("A", "B");
+ Instant watermark1 = Instant.now();
+ PartitionRecord partitionRecord1 =
+ new PartitionRecord(partition1, watermark1, "1", watermark1, Collections.emptyList(), null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord1);
+
+ ByteStringRange partition2 = ByteStringRange.create("B", "D");
+ ChangeStreamContinuationToken partition2Token1 =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "tokenBC");
+ ChangeStreamContinuationToken partition2Token2 =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("C", "D"), "tokenCD");
+ Instant watermark2 = Instant.now();
+ PartitionRecord partitionRecord2 =
+ new PartitionRecord(
+ partition2,
+ Arrays.asList(partition2Token1, partition2Token2),
+ "2",
+ watermark2,
+ Collections.emptyList(),
+ null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord2);
+
+ List<PartitionRecord> partitionRecords = metadataTableDao.readAllStreamPartitions();
+ assertThat(partitionRecords, containsInAnyOrder(partitionRecord1, partitionRecord2));
+
+ // Update the watermark of partition2
+ Instant watermark3 = watermark2.plus(Duration.standardSeconds(10));
+ metadataTableDao.updateWatermark(partition2, watermark3, null);
+ PartitionRecord partitionRecord2UpdatedWatermark =
+ new PartitionRecord(
+ partition2,
+ Arrays.asList(partition2Token1, partition2Token2),
+ "2",
+ watermark3,
+ Collections.emptyList(),
+ null);
+ partitionRecords = metadataTableDao.readAllStreamPartitions();
+ assertThat(
+ partitionRecords, containsInAnyOrder(partitionRecord1, partitionRecord2UpdatedWatermark));
+
+ // Update partition1 with new watermark and continuation token
+ Instant watermark4 = watermark1.plus(Duration.standardMinutes(1));
+ ChangeStreamContinuationToken partition1Token =
+ ChangeStreamContinuationToken.create(partition1, "tokenAB");
+ metadataTableDao.updateWatermark(partition1, watermark4, partition1Token);
+ PartitionRecord partitionRecord1UpdatedWatermark =
+ new PartitionRecord(
+ partition1,
+ Collections.singletonList(partition1Token),
+ "1",
+ watermark4,
+ Collections.emptyList(),
+ null);
+ partitionRecords = metadataTableDao.readAllStreamPartitions();
+ assertThat(
+ partitionRecords,
+ containsInAnyOrder(partitionRecord1UpdatedWatermark, partitionRecord2UpdatedWatermark));
+
+ // Release the lock on partition2, the results should be the same.
+ metadataTableDao.releaseStreamPartitionLockForDeletion(partition2, partitionRecord2.getUuid());
+ partitionRecords = metadataTableDao.readAllStreamPartitions();
+ PartitionRecord partitionRecord2NoUuid =
+ new PartitionRecord(
+ partition2,
+ Arrays.asList(partition2Token1, partition2Token2),
+ watermark3,
+ Collections.emptyList());
+ assertThat(
+ partitionRecords,
+ containsInAnyOrder(partitionRecord1UpdatedWatermark, partitionRecord2NoUuid));
+
+ // Delete partition2
+ metadataTableDao.deleteStreamPartitionRow(partition2);
+ partitionRecords = metadataTableDao.readAllStreamPartitions();
+ assertThat(partitionRecords, containsInAnyOrder(partitionRecord1UpdatedWatermark));
+ }
+
+ @Test
+ public void testUpdateDetectNewPartitionWatermark() {
+ Instant watermark = Instant.now();
+ metadataTableDao.updateDetectNewPartitionWatermark(watermark);
+ Row row =
+ dataClient.readRow(
+ metadataTableAdminDao.getTableId(),
+ metadataTableDao
+ .getChangeStreamNamePrefix()
+ .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX));
+ assertNull(MetadataTableEncoder.parseTokenFromRow(row));
+ assertEquals(watermark, parseWatermarkFromRow(row));
+ }
+
+ @Test
+ public void testReadWriteLowWatermark() {
+ Instant t = Instant.now();
+ metadataTableDao.updateDetectNewPartitionWatermark(t);
+ assertEquals(t, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+ t = t.plus(Duration.standardMinutes(10));
+ metadataTableDao.updateDetectNewPartitionWatermark(t);
+ assertEquals(t, metadataTableDao.readDetectNewPartitionsState().getWatermark());
}
@Test
@@ -312,23 +561,23 @@ public class MetadataTableDaoTest {
dataClient.readRow(
metadataTableAdminDao.getTableId(),
metadataTableDao.convertPartitionToStreamPartitionRowKey(partition));
- assertEquals(token.getToken(), MetadataTableEncoder.getTokenFromRow(row));
- assertEquals(watermark, MetadataTableEncoder.parseWatermarkFromRow(row));
+ assertEquals(token.getToken(), MetadataTableEncoder.parseTokenFromRow(row));
+ assertEquals(watermark, parseWatermarkFromRow(row));
}
@Test
public void readAndWriteValidMissingPartitionsDuration() {
- HashMap<ByteStringRange, Long> missingPartitionsDuration = new HashMap<>();
- missingPartitionsDuration.put(ByteStringRange.create("A", "B"), 100L);
+ HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
+ missingPartitionsDuration.put(ByteStringRange.create("A", "B"), Instant.ofEpochMilli(100));
metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionsDuration);
- HashMap<ByteStringRange, Long> actualMissingPartitionsDuration =
+ HashMap<ByteStringRange, Instant> actualMissingPartitionsDuration =
metadataTableDao.readDetectNewPartitionMissingPartitions();
assertEquals(missingPartitionsDuration, actualMissingPartitionsDuration);
}
@Test
public void readAndWriteInvalidMissingPartitionsDuration() {
- HashMap<ByteStringRange, Long> missingPartitionsDuration = new HashMap<>();
+ HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
RowMutation rowMutation =
RowMutation.create(
@@ -343,16 +592,104 @@ public class MetadataTableDaoTest {
dataClient.mutateRow(rowMutation);
// We should still be able to read the invalid serialization and return an empty map.
- HashMap<ByteStringRange, Long> actualMissingPartitionsDuration =
+ HashMap<ByteStringRange, Instant> actualMissingPartitionsDuration =
metadataTableDao.readDetectNewPartitionMissingPartitions();
assertEquals(missingPartitionsDuration, actualMissingPartitionsDuration);
}
@Test
public void readMissingPartitionsWithoutDNPRow() {
- HashMap<ByteStringRange, Long> missingPartitionsDuration = new HashMap<>();
- HashMap<ByteStringRange, Long> actualMissingPartitionsDuration =
+ HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
+ HashMap<ByteStringRange, Instant> actualMissingPartitionsDuration =
metadataTableDao.readDetectNewPartitionMissingPartitions();
assertEquals(missingPartitionsDuration, actualMissingPartitionsDuration);
}
+
+ @Test
+ public void readMissingPartitionsWithoutMissingPartitionsCell() {
+ HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
+ // Write DNP row but no missing partitions cell.
+ metadataTableDao.updateDetectNewPartitionWatermark(Instant.now());
+ HashMap<ByteStringRange, Instant> actualMissingPartitionsDuration =
+ metadataTableDao.readDetectNewPartitionMissingPartitions();
+ assertEquals(missingPartitionsDuration, actualMissingPartitionsDuration);
+ }
+
+ @Test
+ public void testReleaseStreamPartitionLock() throws InvalidProtocolBufferException {
+ ByteStringRange partition = ByteStringRange.create("A", "B");
+ String uuid = "1234";
+ PartitionRecord partitionRecord =
+ new PartitionRecord(
+ partition, Instant.now(), uuid, Instant.now(), Collections.emptyList(), null);
+ metadataTableDao.lockAndRecordPartition(partitionRecord);
+ assertFalse(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+ assertTrue(metadataTableDao.doHoldLock(partition, uuid));
+
+ assertFalse(metadataTableDao.releaseStreamPartitionLockForDeletion(partition, "0000"));
+ assertFalse(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+ assertTrue(metadataTableDao.doHoldLock(partition, uuid));
+
+ assertTrue(metadataTableDao.releaseStreamPartitionLockForDeletion(partition, uuid));
+ assertTrue(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+ assertFalse(metadataTableDao.doHoldLock(partition, uuid));
+ }
+
+ @Test
+ public void testDeleteStreamPartitionRow() throws InvalidProtocolBufferException {
+ ByteStringRange partition = ByteStringRange.create("A", "B");
+ String uuid = "1234";
+ PartitionRecord partitionRecord =
+ new PartitionRecord(
+ partition, Instant.now(), uuid, Instant.now(), Collections.emptyList(), null);
+ assertTrue(metadataTableDao.lockAndRecordPartition(partitionRecord));
+ metadataTableDao.updateWatermark(partition, Instant.now(), null);
+ assertFalse(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+ assertTrue(metadataTableDao.doHoldLock(partition, uuid));
+
+ // Can't delete it because lock needs to be released first
+ assertFalse(metadataTableDao.deleteStreamPartitionRow(partition));
+ assertFalse(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+ assertTrue(metadataTableDao.doHoldLock(partition, uuid));
+
+ // Release and delete
+ assertTrue(metadataTableDao.releaseStreamPartitionLockForDeletion(partition, uuid));
+ assertTrue(metadataTableDao.deleteStreamPartitionRow(partition));
+ assertTrue(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+ assertFalse(metadataTableDao.doHoldLock(partition, uuid));
+ }
+
+ @Test
+ public void testLockPartitionRecordsMetadata() {
+ ByteStringRange partition = ByteStringRange.create("A", "C");
+ ChangeStreamContinuationToken token =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "token1");
+ PartitionRecord partitionRecord =
+ new PartitionRecord(
+ partition,
+ Collections.singletonList(token),
+ "1234",
+ Instant.now(),
+ Collections.emptyList(),
+ Instant.now());
+ metadataTableDao.lockAndRecordPartition(partitionRecord);
+ assertTrue(metadataTableDao.doHoldLock(partition, "1234"));
+ // Check that watermark and tokens are correctly written.
+ Row row =
+ dataClient.readRow(
+ metadataTableAdminDao.getTableId(),
+ metadataTableDao.convertPartitionToStreamPartitionRowKey(partition));
+ assertTrue(
+ row.getCells(
+ MetadataTableAdminDao.CF_SHOULD_DELETE, MetadataTableAdminDao.QUALIFIER_DEFAULT)
+ .isEmpty());
+ assertEquals(partitionRecord.getParentLowWatermark(), parseWatermarkFromRow(row));
+ assertEquals(
+ token.toByteString(),
+ row.getCells(
+ MetadataTableAdminDao.CF_INITIAL_TOKEN,
+ ByteStringRange.serializeToByteString(token.getPartition()))
+ .get(0)
+ .getValue());
+ }
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java
index 0807719c8cb..a9fa54d4c49 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java
@@ -17,11 +17,11 @@
*/
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -31,14 +31,18 @@ import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
-import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
+import com.google.protobuf.ByteString;
import java.io.IOException;
+import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO;
+import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO.ExistingPipelineOptions;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs;
+import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -56,7 +60,7 @@ public class InitializeDoFnTest {
@Mock private DaoFactory daoFactory;
@Mock private transient MetadataTableAdminDao metadataTableAdminDao;
private transient MetadataTableDao metadataTableDao;
- @Mock private DoFn.OutputReceiver<Instant> outputReceiver;
+ @Mock private DoFn.OutputReceiver<InitialPipelineState> outputReceiver;
private final String tableId = "table";
private static BigtableDataClient dataClient;
@@ -83,6 +87,7 @@ public class InitializeDoFnTest {
String changeStreamName = "changeStreamName";
metadataTableAdminDao =
spy(new MetadataTableAdminDao(adminClient, null, changeStreamName, tableId));
+ metadataTableAdminDao.createMetadataTable();
doReturn(true).when(metadataTableAdminDao).isAppProfileSingleClusterAndTransactional(any());
when(daoFactory.getMetadataTableAdminDao()).thenReturn(metadataTableAdminDao);
metadataTableDao =
@@ -95,29 +100,176 @@ public class InitializeDoFnTest {
@Test
public void testInitializeDefault() throws IOException {
Instant startTime = Instant.now();
- InitializeDoFn initializeDoFn = new InitializeDoFn(daoFactory, "app-profile", startTime);
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory,
+ "app-profile",
+ startTime,
+ BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS);
initializeDoFn.processElement(outputReceiver);
- verify(outputReceiver, times(1)).output(startTime);
- assertTrue(adminClient.exists(tableId));
- Row row =
- dataClient.readRow(
- tableId,
- metadataTableAdminDao
- .getChangeStreamNamePrefix()
- .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX));
- assertNotNull(row);
- assertEquals(
- 1,
- row.getCells(MetadataTableAdminDao.CF_VERSION, MetadataTableAdminDao.QUALIFIER_DEFAULT)
- .size());
- assertEquals(
- MetadataTableAdminDao.CURRENT_METADATA_TABLE_VERSION,
- (int)
- Longs.fromByteArray(
- row.getCells(
- MetadataTableAdminDao.CF_VERSION, MetadataTableAdminDao.QUALIFIER_DEFAULT)
- .get(0)
- .getValue()
- .toByteArray()));
+ verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+ }
+
+ @Test
+ public void testInitializeStopWithExistingPipeline() throws IOException {
+ metadataTableDao.updateDetectNewPartitionWatermark(Instant.now());
+ Instant startTime = Instant.now();
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory,
+ "app-profile",
+ startTime,
+ BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS);
+ initializeDoFn.processElement(outputReceiver);
+ verify(outputReceiver, never()).output(any());
+ }
+
+ @Test
+ public void testInitializeStopWithoutDNP() throws IOException {
+ // DNP row doesn't exist, so we don't need to stop the pipeline. But some random data row with
+ // the same prefix exists. We want to make sure we clean it up even in "STOP" option.
+ dataClient.mutateRow(
+ RowMutation.create(
+ tableId,
+ metadataTableAdminDao
+ .getChangeStreamNamePrefix()
+ .concat(ByteString.copyFromUtf8("existing_row")))
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+ Instant startTime = Instant.now();
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory,
+ "app-profile",
+ startTime,
+ BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS);
+ initializeDoFn.processElement(outputReceiver);
+ verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+ assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix()));
+ }
+
+ @Test
+ public void testInitializeNewWithoutDNP() throws IOException {
+ // DNP row doesn't exist, so we output with provided start time and we clean up any existing
+ // rows under the prefix.
+ dataClient.mutateRow(
+ RowMutation.create(
+ tableId,
+ metadataTableAdminDao
+ .getChangeStreamNamePrefix()
+ .concat(ByteString.copyFromUtf8("existing_row")))
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+ Instant startTime = Instant.now();
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.NEW);
+ initializeDoFn.processElement(outputReceiver);
+ verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+ assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix()));
+ }
+
+ @Test
+ public void testInitializeNewWithDNP() throws IOException {
+ metadataTableDao.updateDetectNewPartitionWatermark(Instant.now());
+ dataClient.mutateRow(
+ RowMutation.create(
+ tableId,
+ metadataTableAdminDao
+ .getChangeStreamNamePrefix()
+ .concat(ByteString.copyFromUtf8("existing_row")))
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+ Instant startTime = Instant.now();
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.NEW);
+ initializeDoFn.processElement(outputReceiver);
+ verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+ assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix()));
+ }
+
+ @Test
+ public void testInitializeResumeWithoutDNP() throws IOException {
+ dataClient.mutateRow(
+ RowMutation.create(
+ tableId,
+ metadataTableAdminDao
+ .getChangeStreamNamePrefix()
+ .concat(ByteString.copyFromUtf8("existing_row")))
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+ Instant startTime = Instant.now();
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.RESUME_OR_NEW);
+ initializeDoFn.processElement(outputReceiver);
+ // We want to resume but there's no DNP row, so we resume from the startTime provided.
+ verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+ }
+
+ @Test
+ public void testInitializeResumeWithDNP() throws IOException {
+ Instant resumeTime = Instant.now().minus(Duration.standardSeconds(10000));
+ metadataTableDao.updateDetectNewPartitionWatermark(resumeTime);
+ dataClient.mutateRow(
+ RowMutation.create(
+ tableId,
+ metadataTableAdminDao
+ .getChangeStreamNamePrefix()
+ .concat(ByteString.copyFromUtf8("existing_row")))
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+ Instant startTime = Instant.now();
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.RESUME_OR_NEW);
+ initializeDoFn.processElement(outputReceiver);
+ verify(outputReceiver, times(1)).output(new InitialPipelineState(resumeTime, true));
+ assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix()));
+ }
+
+ @Test
+ public void testInitializeSkipCleanupWithoutDNP() throws IOException {
+ ByteString metadataRowKey =
+ metadataTableAdminDao
+ .getChangeStreamNamePrefix()
+ .concat(ByteString.copyFromUtf8("existing_row"));
+ dataClient.mutateRow(
+ RowMutation.create(tableId, metadataRowKey)
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+ Instant startTime = Instant.now();
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory, "app-profile", startTime, ExistingPipelineOptions.SKIP_CLEANUP);
+ initializeDoFn.processElement(outputReceiver);
+ // Skip cleanup will always resume from startTime
+ verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+ // Existing metadata shouldn't be cleaned up
+ assertNotNull(dataClient.readRow(tableId, metadataRowKey));
+ }
+
+ @Test
+ public void testInitializeSkipCleanupWithDNP() throws IOException {
+ Instant resumeTime = Instant.now().minus(Duration.standardSeconds(10000));
+ metadataTableDao.updateDetectNewPartitionWatermark(resumeTime);
+ ByteString metadataRowKey =
+ metadataTableAdminDao
+ .getChangeStreamNamePrefix()
+ .concat(ByteString.copyFromUtf8("existing_row"));
+ dataClient.mutateRow(
+ RowMutation.create(tableId, metadataRowKey)
+ .setCell(
+ MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+ Instant startTime = Instant.now();
+ InitializeDoFn initializeDoFn =
+ new InitializeDoFn(
+ daoFactory, "app-profile", startTime, ExistingPipelineOptions.SKIP_CLEANUP);
+ initializeDoFn.processElement(outputReceiver);
+ // We don't want the pipeline to resume to avoid duplicates
+ verify(outputReceiver, never()).output(any());
+ // Existing metadata shouldn't be cleaned up
+ assertNotNull(dataClient.readRow(tableId, metadataRowKey));
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java
index 38aae227787..e8f59c2bd17 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java
@@ -21,7 +21,6 @@ import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConvert
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyObject;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -34,6 +33,7 @@ import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord;
import com.google.cloud.bigtable.data.v2.models.Range;
import com.google.protobuf.ByteString;
import java.io.IOException;
+import java.util.Collections;
import java.util.Iterator;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ActionFactory;
@@ -95,8 +95,7 @@ public class ReadChangeStreamPartitionDoFnTest {
@Test
public void testProcessElementAndGetSize() throws IOException, InterruptedException {
long watermarkLag = 10;
- Instant tenSecondsAgo =
- Instant.now().minus(org.joda.time.Duration.standardSeconds(watermarkLag));
+ Instant tenSecondsAgo = Instant.now().minus(Duration.standardSeconds(watermarkLag));
Range.ByteStringRange partitionRange = Range.ByteStringRange.create("", "");
ChangeStreamContinuationToken testToken =
ChangeStreamContinuationToken.create(partitionRange, "test");
@@ -106,6 +105,7 @@ public class ReadChangeStreamPartitionDoFnTest {
tenSecondsAgo,
"uid-a",
tenSecondsAgo,
+ Collections.emptyList(),
Instant.now().plus(Duration.standardSeconds(60)));
long mutationSize = 100L;
when(sizeEstimator.sizeOf(any())).thenReturn(mutationSize);
@@ -124,14 +124,13 @@ public class ReadChangeStreamPartitionDoFnTest {
when(mockMutation.getToken()).thenReturn(testToken.getToken());
when(mockMutation.getCommitTimestamp()).thenReturn(toThreetenInstant(tenSecondsAgo));
- when(metadataTableDao.lockPartition(any(), any())).thenReturn(true);
+ when(metadataTableDao.lockAndRecordPartition(any())).thenReturn(true);
ServerStream<ChangeStreamRecord> mockStream = mock(ServerStream.class);
Iterator<ChangeStreamRecord> mockResponses = mock(Iterator.class);
when(mockResponses.hasNext()).thenReturn(true, true, true);
when(mockResponses.next()).thenReturn(mockMutation, mockMutation, mockMutation);
when(mockStream.iterator()).thenReturn(mockResponses);
- when(changeStreamDao.readChangeStreamPartition(
- anyObject(), anyObject(), anyObject(), anyObject(), anyBoolean()))
+ when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any(), anyBoolean()))
.thenReturn(mockStream);
when(watermarkEstimator.getState()).thenReturn(tenSecondsAgo);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleanerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleanerTest.java
new file mode 100644
index 00000000000..27294911b45
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleanerTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Test;
+
+public class OrphanedMetadataCleanerTest {
+
+ @Test
+ public void testOrphanedNewPartitions() {
+ OrphanedMetadataCleaner orphanedMetadataCleaner = new OrphanedMetadataCleaner();
+ // Not waiting long enough
+ NewPartition newPartition1 =
+ new NewPartition(
+ ByteStringRange.create("A", "B"),
+ Collections.emptyList(),
+ Instant.EPOCH,
+ Instant.now().minus(Duration.standardMinutes(10)));
+ orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition1);
+ // Orphaned long enough. Should be cleaned up
+ NewPartition newPartition2 =
+ new NewPartition(
+ ByteStringRange.create("B", "C"),
+ Collections.emptyList(),
+ Instant.EPOCH,
+ Instant.now().minus(Duration.standardMinutes(15)).minus(Duration.standardSeconds(1)));
+ orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition2);
+ // Just on the edge of not orphaned long enough
+ NewPartition newPartition3 =
+ new NewPartition(
+ ByteStringRange.create("C", "D"),
+ Collections.emptyList(),
+ Instant.EPOCH,
+ Instant.now().minus(Duration.standardMinutes(15)).plus(Duration.standardSeconds(1)));
+ orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition3);
+ // Orphaned long enough, but it overlaps with a missing partition, so not cleaned up.
+ NewPartition newPartition4 =
+ new NewPartition(
+ ByteStringRange.create("D", "F"),
+ Collections.emptyList(),
+ Instant.EPOCH,
+ Instant.now().minus(Duration.standardMinutes(15)).minus(Duration.standardSeconds(1)));
+ orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition4);
+ List<ByteStringRange> missingPartitions = new ArrayList<>();
+
+ // Haven't called addMissingPartitions, so we don't know if there are any missing partitions.
+ assertTrue(orphanedMetadataCleaner.getOrphanedNewPartitions().isEmpty());
+
+ missingPartitions.add(ByteStringRange.create("E", "G"));
+ missingPartitions.add(ByteStringRange.create("H", "I"));
+ orphanedMetadataCleaner.addMissingPartitions(missingPartitions);
+
+ assertEquals(
+ Collections.singletonList(newPartition2),
+ orphanedMetadataCleaner.getOrphanedNewPartitions());
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java
index 63c46e6b87f..aab2bdc8f64 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java
@@ -17,41 +17,56 @@
*/
package org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertEquals;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
-import com.google.protobuf.ByteString;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.List;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.joda.time.Duration;
import org.joda.time.Instant;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+@RunWith(MockitoJUnitRunner.class)
public class PartitionReconcilerTest {
@ClassRule
public static final BigtableEmulatorRule BIGTABLE_EMULATOR_RULE = BigtableEmulatorRule.create();
- private static final long MORE_THAN_FIVE_MINUTES_MILLI = 5 * 60 * 1000L + 1L;
+ private static final Duration MORE_THAN_ONE_MINUTE = Duration.standardSeconds(61);
+ private static final Duration MORE_THAN_TEN_MINUTES = Duration.standardSeconds(10 * 60 + 1);
private MetadataTableDao metadataTableDao;
private static BigtableDataClient dataClient;
private static BigtableTableAdminClient adminClient;
+ private Instant lowWatermark;
+ private Instant startTime;
+
+ @Mock private ChangeStreamMetrics metrics;
+
@BeforeClass
public static void beforeClass() throws IOException {
BigtableTableAdminSettings adminSettings =
@@ -80,6 +95,10 @@ public class PartitionReconcilerTest {
dataClient,
metadataTableAdminDao.getTableId(),
metadataTableAdminDao.getChangeStreamNamePrefix());
+ lowWatermark = Instant.now();
+ // We set startTime to 30 min prior so if we reconcile without token, we use don't use an hour
+ // prior but the startTime.
+ startTime = lowWatermark.minus(Duration.standardMinutes(30));
}
// [A-B) is missing because
@@ -89,41 +108,54 @@ public class PartitionReconcilerTest {
@Test
public void testNewMissingMergePartitionIsNotReconciled() {
// No reconciliation should happen because the missing partition hasn't been waiting for more
- // than 5 minutes.
+ // than 1 minute.
ByteStringRange partitionAB = ByteStringRange.create("A", "B");
ByteStringRange partitionAC = ByteStringRange.create("A", "C");
- PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao);
+ ChangeStreamContinuationToken tokenAB = ChangeStreamContinuationToken.create(partitionAB, "AB");
+ PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAB));
// Since [A-B) received the CloseStream merge, there should be an NewPartitions row for [A-C)
// with [A-B) as a parent.
- partitionReconciler.addNewPartition(partitionAC, ByteString.copyFromUtf8("FakeRowKeyForAC"));
- HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile =
- partitionReconciler.getPartitionsToReconcile();
- assertEquals(0, partitionsToReconcile.size());
+ NewPartition newPartitionAC =
+ new NewPartition(partitionAC, Collections.singletonList(tokenAB), Instant.now());
+ partitionReconciler.addIncompleteNewPartitions(newPartitionAC);
+ List<PartitionRecord> reconciledPartitions =
+ partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+ assertEquals(0, reconciledPartitions.size());
}
+ // [A-B) merging into [A-C)
+ // [B-C) did not need to merge into [A-C)
+ // Reconciler should, after 1 minute, output partition [A-B) with continuation token.
@Test
public void testLongMissingMergePartitionIsReconciled() {
ByteStringRange partitionAB = ByteStringRange.create("A", "B");
ByteStringRange partitionAC = ByteStringRange.create("A", "C");
+ ChangeStreamContinuationToken tokenAB = ChangeStreamContinuationToken.create(partitionAB, "AB");
- // Artificially create that partitionAB has been missing for more than 5 minutes.
- HashMap<ByteStringRange, Long> missingPartitionDurations = new HashMap<>();
- missingPartitionDurations.put(
- partitionAB, Instant.now().getMillis() - MORE_THAN_FIVE_MINUTES_MILLI);
+ // Artificially create that partitionAB has been missing for more than 1 minute.
+ HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+ missingPartitionDurations.put(partitionAB, Instant.now().minus(MORE_THAN_ONE_MINUTE));
metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
- PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao);
+ PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAB));
// Since [A-B) received the CloseStream merge, there should be an NewPartitions row for [A-C)
// with [A-B) as a parent.
- partitionReconciler.addNewPartition(partitionAC, ByteString.copyFromUtf8("FakeRowKeyForAC"));
- HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile =
- partitionReconciler.getPartitionsToReconcile();
- assertEquals(partitionsToReconcile.keySet(), Collections.singleton(partitionAB));
- assertEquals(
- partitionsToReconcile.get(partitionAB),
- Collections.singleton(ByteString.copyFromUtf8("FakeRowKeyForAC")));
+ NewPartition newPartitionAC =
+ new NewPartition(partitionAC, Collections.singletonList(tokenAB), Instant.now());
+ partitionReconciler.addIncompleteNewPartitions(newPartitionAC);
+ List<PartitionRecord> reconciledPartitions =
+ partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+
+ PartitionRecord expectedRecord =
+ new PartitionRecord(
+ partitionAB,
+ Collections.singletonList(tokenAB),
+ lowWatermark,
+ Collections.singletonList(newPartitionAC));
+ assertEquals(1, reconciledPartitions.size());
+ assertEquals(expectedRecord, reconciledPartitions.get(0));
}
// [A-B) merges into [A-C)
@@ -134,26 +166,47 @@ public class PartitionReconcilerTest {
ByteStringRange partitionAC = ByteStringRange.create("A", "C");
ByteStringRange partitionAD = ByteStringRange.create("A", "D");
- // Artificially create that partitionAD has been missing for more than 5 minutes.
- HashMap<ByteStringRange, Long> missingPartitionDurations = new HashMap<>();
- missingPartitionDurations.put(
- partitionAD, Instant.now().getMillis() - MORE_THAN_FIVE_MINUTES_MILLI);
+ ChangeStreamContinuationToken tokenAB =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+ ChangeStreamContinuationToken tokenBC =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
+ ChangeStreamContinuationToken tokenCD =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("C", "D"), "CD");
+ // Artificially create that partitionAD has been missing for more than 10 minutes.
+ HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+ missingPartitionDurations.put(partitionAD, Instant.now().minus(MORE_THAN_TEN_MINUTES));
metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
- PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao);
+ PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
// We are not missing A-B, B-C, C-D, from the metadata table view, we are simply missing A-D.
partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAD));
- partitionReconciler.addNewPartition(partitionAC, ByteString.copyFromUtf8("FakeRowKeyForAC"));
- partitionReconciler.addNewPartition(partitionAD, ByteString.copyFromUtf8("FakeRowKeyForAD"));
- HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile =
- partitionReconciler.getPartitionsToReconcile();
- assertEquals(partitionsToReconcile.keySet(), Collections.singleton(partitionAD));
- assertEquals(
- partitionsToReconcile.get(partitionAD),
- new HashSet<>(
- Arrays.asList(
- ByteString.copyFromUtf8("FakeRowKeyForAC"),
- ByteString.copyFromUtf8("FakeRowKeyForAD"))));
+ // AB merging into AC
+ NewPartition newPartitionAC =
+ new NewPartition(partitionAC, Collections.singletonList(tokenAB), Instant.now());
+ partitionReconciler.addIncompleteNewPartitions(newPartitionAC);
+ // BC and CD merging into AD
+ NewPartition newPartitionAD =
+ new NewPartition(partitionAD, Arrays.asList(tokenBC, tokenCD), Instant.now());
+ partitionReconciler.addIncompleteNewPartitions(newPartitionAD);
+
+ List<PartitionRecord> reconciledPartitions =
+ partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+ // As part of reconciling we split up the parents into their own NewPartition, so they can be
+ // cleaned up separately.
+ NewPartition newPartitionADWithBC =
+ new NewPartition(
+ partitionAD, Collections.singletonList(tokenBC), newPartitionAD.getLowWatermark());
+ NewPartition newPartitionADWithCD =
+ new NewPartition(
+ partitionAD, Collections.singletonList(tokenCD), newPartitionAD.getLowWatermark());
+ PartitionRecord expectedRecord =
+ new PartitionRecord(
+ partitionAD,
+ Arrays.asList(tokenAB, tokenBC, tokenCD),
+ lowWatermark,
+ Arrays.asList(newPartitionAC, newPartitionADWithBC, newPartitionADWithCD));
+ assertEquals(1, reconciledPartitions.size());
+ assertEquals(expectedRecord, reconciledPartitions.get(0));
}
// [A-B) merges into [A-D)
@@ -161,35 +214,123 @@ public class PartitionReconcilerTest {
// We're missing [A-C).
// [A-B) and [B-D) were supposed to merge into [A-D)
// Before [B-D) received the CloseStream merge, [A-D) actually splits into [A-C) and [C-D) which
- // is why [B-D) received CloseStream split into [A-C) and [C-D)
- // Now we need to merge [A-B) and [B-D) into [A-C). [C-D) part of the split was successful on its
- // own.
+ // is why [B-D) received CloseStream split into [A-C) and [C-D). Note that the tokens in the
+ // CloseStream are for BC and CD. So when performing reconciliation, we're reconciling BC, not BD.
+ // Reconciler should merge [A-B) and [B-C) into [A-C). [C-D) part of the split was successful.
@Test
public void testMismatchedMergeSplitPartitionIsReconciled() {
ByteStringRange partitionAC = ByteStringRange.create("A", "C");
ByteStringRange partitionAD = ByteStringRange.create("A", "D");
+ ChangeStreamContinuationToken tokenAB =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+ ChangeStreamContinuationToken tokenBC =
+ ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
- // Artificially create that partitionAC has been missing for more than 5 minutes.
- HashMap<ByteStringRange, Long> missingPartitionDurations = new HashMap<>();
- missingPartitionDurations.put(
- partitionAC, Instant.now().getMillis() - MORE_THAN_FIVE_MINUTES_MILLI);
+ // Artificially create that partitionAC has been missing for more than 10 minutes.
+ HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+ missingPartitionDurations.put(partitionAC, Instant.now().minus(MORE_THAN_TEN_MINUTES));
metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
- PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao);
- // We are not missing A-B, B-D, from the metadata table view, we are simply missing A-C.
+ PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
+ // We are not missing A-B, B-C, from the metadata table view, we are simply missing A-C.
partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAC));
// A-B tried to merge into A-D
- partitionReconciler.addNewPartition(partitionAD, ByteString.copyFromUtf8("FakeRowKeyForAD"));
- // B-D tried to split/merge into A-C
- partitionReconciler.addNewPartition(partitionAC, ByteString.copyFromUtf8("FakeRowKeyForAC"));
- HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile =
- partitionReconciler.getPartitionsToReconcile();
- assertEquals(partitionsToReconcile.keySet(), Collections.singleton(partitionAC));
- assertEquals(
- partitionsToReconcile.get(partitionAC),
- new HashSet<>(
- Arrays.asList(
- ByteString.copyFromUtf8("FakeRowKeyForAD"),
- ByteString.copyFromUtf8("FakeRowKeyForAC"))));
+ NewPartition newPartitionAD =
+ new NewPartition(partitionAD, Collections.singletonList(tokenAB), Instant.now());
+ partitionReconciler.addIncompleteNewPartitions(newPartitionAD);
+ // B-C tried to split/merge into A-C
+ NewPartition newPartitionAC =
+ new NewPartition(partitionAC, Collections.singletonList(tokenBC), Instant.now());
+ partitionReconciler.addIncompleteNewPartitions(newPartitionAC);
+ List<PartitionRecord> reconciledPartitions =
+ partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+
+ PartitionRecord expectedRecord =
+ new PartitionRecord(
+ partitionAC,
+ Arrays.asList(tokenAB, tokenBC),
+ lowWatermark,
+ Arrays.asList(newPartitionAD, newPartitionAC));
+ assertEquals(1, reconciledPartitions.size());
+ assertEquals(expectedRecord, reconciledPartitions.get(0));
+ }
+
+ // AB is missing without any tokens for more than 10 minutes, reconciler will restart the
+ // partition with low watermark - 1 hour.
+ @Test
+ public void testMissingPartitionWithoutToken() {
+ ByteStringRange partitionAB = ByteStringRange.create("A", "B");
+
+ // Artificially create that partitionAB has been missing for more than 10 minutes.
+ HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+ missingPartitionDurations.put(partitionAB, Instant.now().minus(MORE_THAN_TEN_MINUTES));
+ metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
+
+ PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
+ partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAB));
+
+ List<PartitionRecord> reconciledPartitions =
+ partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+
+ PartitionRecord expectedRecord =
+ new PartitionRecord(partitionAB, startTime, lowWatermark, Collections.emptyList());
+ assertEquals(1, reconciledPartitions.size());
+ assertEquals(expectedRecord, reconciledPartitions.get(0));
+ }
+
+ // We're missing AD but we only have partition AB and CD. We should reconcile by outputting AB and
+ // CD and then create a new partition for BC with start_time = low watermark - 1 hour
+ @Test
+ public void testMissingPartitionWithSomeToken() {
+ ByteStringRange partitionAD = ByteStringRange.create("A", "D");
+ ByteStringRange partitionBC = ByteStringRange.create("B", "C");
+ ByteStringRange partitionAB = ByteStringRange.create("A", "B");
+ ChangeStreamContinuationToken tokenAB = ChangeStreamContinuationToken.create(partitionAB, "AB");
+ ByteStringRange partitionCD = ByteStringRange.create("C", "D");
+ ChangeStreamContinuationToken tokenCD = ChangeStreamContinuationToken.create(partitionCD, "CD");
+
+ // Artificially create that partitionAD has been missing for more than 10 minutes.
+ HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+ missingPartitionDurations.put(partitionAD, Instant.now().minus(MORE_THAN_TEN_MINUTES));
+ metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
+
+ PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
+ partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAD));
+
+ NewPartition newPartitionAD =
+ new NewPartition(partitionAD, Arrays.asList(tokenAB, tokenCD), Instant.now());
+ partitionReconciler.addIncompleteNewPartitions(newPartitionAD);
+
+ List<PartitionRecord> reconciledPartitions =
+ partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+
+ // As part of reconciling we split up the parents into their own NewPartition, so they can be
+ // cleaned up separately.
+ NewPartition newPartitionADWithAB =
+ new NewPartition(
+ partitionAD, Collections.singletonList(tokenAB), newPartitionAD.getLowWatermark());
+ NewPartition newPartitionADWithCD =
+ new NewPartition(
+ partitionAD, Collections.singletonList(tokenCD), newPartitionAD.getLowWatermark());
+ PartitionRecord expectedRecordAB =
+ new PartitionRecord(
+ partitionAB,
+ Collections.singletonList(tokenAB),
+ lowWatermark,
+ Collections.singletonList(newPartitionADWithAB));
+ PartitionRecord expectedRecordCD =
+ new PartitionRecord(
+ partitionCD,
+ Collections.singletonList(tokenCD),
+ lowWatermark,
+ Collections.singletonList(newPartitionADWithCD));
+ PartitionRecord expectedRecordBC =
+ new PartitionRecord(partitionBC, startTime, lowWatermark, Collections.emptyList());
+ assertEquals(3, reconciledPartitions.size());
+
+ assertThat(
+ reconciledPartitions,
+ containsInAnyOrder(
+ Arrays.asList(expectedRecordAB, expectedRecordBC, expectedRecordCD).toArray()));
}
}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTrackerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTrackerTest.java
new file mode 100644
index 00000000000..42200e8c359
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTrackerTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.beam.sdk.io.range.OffsetRange;
+import org.apache.beam.sdk.transforms.splittabledofn.SplitResult;
+import org.junit.Test;
+
+public class DetectNewPartitionsTrackerTest {
+
+ @Test
+ public void testTrySplit() {
+ final DetectNewPartitionsTracker tracker = new DetectNewPartitionsTracker(0);
+ assertNull(tracker.trySplit(.7));
+ assertThrows(IllegalStateException.class, tracker::checkDone);
+
+ assertTrue(tracker.tryClaim(tracker.currentRestriction().getFrom()));
+ SplitResult<OffsetRange> checkpointResult = tracker.trySplit(0);
+ tracker.checkDone(); // should not throw exception
+ assertEquals(0L, checkpointResult.getPrimary().getFrom());
+ assertEquals(1L, checkpointResult.getPrimary().getTo());
+ assertEquals(1L, checkpointResult.getResidual().getFrom());
+ assertEquals(Long.MAX_VALUE, checkpointResult.getResidual().getTo());
+ }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java
index f75986bc419..b0bc8de7c81 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
import com.google.cloud.bigtable.data.v2.models.Range;
@@ -54,7 +55,7 @@ public class ReadChangeStreamPartitionProgressTrackerTest {
tracker.currentRestriction().getEstimatedLowWatermark());
try {
tracker.checkDone();
- assertFalse("Should not reach here because checkDone should have thrown an exception", false);
+ fail("Should not reach here because checkDone should have thrown an exception");
} catch (IllegalStateException e) {
assertTrue("There's more work to be done. CheckDone threw an exception", true);
}
@@ -98,6 +99,6 @@ public class ReadChangeStreamPartitionProgressTrackerTest {
streamProgress.setFailToLock(false);
ReadChangeStreamPartitionProgressTracker tracker =
new ReadChangeStreamPartitionProgressTracker(streamProgress);
- assertThrows(java.lang.IllegalStateException.class, tracker::checkDone);
+ assertThrows(IllegalStateException.class, tracker::checkDone);
}
}