You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2023/05/17 03:16:35 UTC

[beam] branch master updated: Update Bigtable Change Stream Connector to include latest features (#26454)

This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new d5c89523938 Update Bigtable Change Stream Connector to include latest features (#26454)
d5c89523938 is described below

commit d5c895239385359edac3b4b96304f1b5507f2774
Author: Tony Tang <nf...@gmail.com>
AuthorDate: Tue May 16 23:16:26 2023 -0400

    Update Bigtable Change Stream Connector to include latest features (#26454)
    
    * Include latest changes from local development
    
    * Reconciler update to reuse continuation tokens to reduce the
      likelihood of missing mutations
    * StreamPartition and NewPartition metadata are performed as 2 phase
      deletes to reduce race conditions that loses tokens
    * Pipelines can be drained and restarted using the same
      ChangeStreamName
    * Added OrphanCleaner to clean up lingering NewPartition metadata
    * General refactoring to improve readability
    
    * Fix javadoc format issue
---
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java       |  84 ++-
 .../changestreams/ByteStringRangeHelper.java       | 175 ++++---
 .../changestreams/ChangeStreamMetrics.java         | 111 ++--
 .../bigtable/changestreams/TimestampConverter.java |   4 +
 .../bigtable/changestreams/UniqueIdGenerator.java  |  19 +-
 .../changestreams/action/ActionFactory.java        |  35 +-
 .../changestreams/action/ChangeStreamAction.java   |   6 +-
 .../action/DetectNewPartitionsAction.java          | 406 ++++++---------
 .../action/GenerateInitialPartitionsAction.java    |  30 +-
 .../action/ProcessNewPartitionsAction.java         | 116 +++++
 .../action/ReadChangeStreamPartitionAction.java    | 101 +++-
 .../action/ResumeFromPreviousPipelineAction.java   |  90 ++++
 .../dao/BigtableChangeStreamAccessor.java          |   9 -
 .../changestreams/dao/ChangeStreamDao.java         |   1 -
 .../gcp/bigtable/changestreams/dao/DaoFactory.java |   7 +-
 .../changestreams/dao/MetadataTableAdminDao.java   |  36 +-
 .../changestreams/dao/MetadataTableDao.java        | 535 ++++++++++++++++---
 .../dofn/DetectNewPartitionsDoFn.java              |  32 +-
 .../changestreams/dofn/InitializeDoFn.java         |  74 ++-
 .../encoder/MetadataTableEncoder.java              |  66 ++-
 .../model/DetectNewPartitionsState.java            |  78 +++
 .../changestreams/model/InitialPipelineState.java  |  70 +++
 .../bigtable/changestreams/model/NewPartition.java | 146 ++++++
 .../changestreams/model/PartitionRecord.java       |  84 +--
 .../model/StreamPartitionWithWatermark.java        |  76 +++
 .../reconciler/OrphanedMetadataCleaner.java        |  87 ++++
 .../reconciler/PartitionReconciler.java            | 183 +++++--
 .../restriction/DetectNewPartitionsTracker.java    |  53 ++
 .../ReadChangeStreamPartitionProgressTracker.java  |  11 +-
 .../changestreams/restriction/StreamProgress.java  |   6 +-
 .../changestreams/ByteStringRangeHelperTest.java   | 262 +++++++---
 .../changestreams/TimestampConverterTest.java      |   6 +
 .../action/ChangeStreamActionTest.java             |   8 +-
 .../action/DetectNewPartitionsActionTest.java      | 570 ++++++++++++++++-----
 .../GenerateInitialPartitionsActionTest.java       |  22 +-
 .../ReadChangeStreamPartitionActionTest.java       | 184 +++++--
 .../ResumeFromPreviousPipelineActionTest.java      | 288 +++++++++++
 .../changestreams/dao/MetadataTableDaoTest.java    | 479 ++++++++++++++---
 .../changestreams/dofn/InitializeDoFnTest.java     | 208 +++++++-
 .../dofn/ReadChangeStreamPartitionDoFnTest.java    |  11 +-
 .../reconciler/OrphanedMetadataCleanerTest.java    |  82 +++
 .../reconciler/PartitionReconcilerTest.java        | 259 +++++++---
 .../DetectNewPartitionsTrackerTest.java            |  45 ++
 ...adChangeStreamPartitionProgressTrackerTest.java |   5 +-
 44 files changed, 4092 insertions(+), 1068 deletions(-)

diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
index 88e94f03140..fc8b5e78e8a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableIO.java
@@ -1782,6 +1782,27 @@ public class BigtableIO {
           cause);
     }
   }
+  /**
+   * Overwrite options to determine what to do if change stream name is being reused and there
+   * exists metadata of the same change stream name.
+   */
+  public enum ExistingPipelineOptions {
+    // Don't start if there exists metadata of the same change stream name.
+    FAIL_IF_EXISTS,
+    // Pick up from where the previous pipeline left off. This will perform resumption at best
+    // effort guaranteeing, at-least-once delivery. So it's likely that duplicate data, seen before
+    // the pipeline was stopped, will be outputted. If previous pipeline doesn't exist, start a new
+    // pipeline.
+    RESUME_OR_NEW,
+    // Same as RESUME_OR_NEW except if previous pipeline doesn't exist, don't start.
+    RESUME_OR_FAIL,
+    // Start a new pipeline. Overriding existing pipeline with the same name.
+    NEW,
+    // This skips cleaning up previous pipeline metadata and starts a new pipeline. This should
+    // only be used to skip cleanup in tests
+    @VisibleForTesting
+    SKIP_CLEANUP,
+  }
 
   @AutoValue
   public abstract static class ReadChangeStream
@@ -1809,11 +1830,14 @@ public class BigtableIO {
 
     abstract @Nullable String getChangeStreamName();
 
+    abstract @Nullable ExistingPipelineOptions getExistingPipelineOptions();
+
     abstract BigtableConfig getMetadataTableBigtableConfig();
 
     abstract @Nullable String getMetadataTableId();
 
     abstract ReadChangeStream.Builder toBuilder();
+
     /**
      * Returns a new {@link BigtableIO.ReadChangeStream} that will stream from the Cloud Bigtable
      * project indicated by given parameter, requires {@link #withInstanceId} to be called to
@@ -1904,6 +1928,17 @@ public class BigtableIO {
       return toBuilder().setChangeStreamName(changeStreamName).build();
     }
 
+    /**
+     * Returns a new {@link BigtableIO.ReadChangeStream} that decides what to do if an existing
+     * pipeline exists with the same change stream name.
+     *
+     * <p>Does not modify this object.
+     */
+    public ReadChangeStream withExistingPipelineOptions(
+        ExistingPipelineOptions existingPipelineOptions) {
+      return toBuilder().setExistingPipelineOptions(existingPipelineOptions).build();
+    }
+
     /**
      * Returns a new {@link BigtableIO.ReadChangeStream} that will use the Cloud Bigtable project
      * indicated by given parameter to manage the metadata of the stream.
@@ -1966,30 +2001,38 @@ public class BigtableIO {
 
     @Override
     public PCollection<KV<ByteString, ChangeStreamMutation>> expand(PBegin input) {
-      checkArgument(getBigtableConfig() != null);
-      checkArgument(getBigtableConfig().getProjectId() != null);
-      checkArgument(getBigtableConfig().getInstanceId() != null);
-      checkArgument(getBigtableConfig().getAppProfileId() != null);
-      checkArgument(getTableId() != null);
+      checkArgument(
+          getBigtableConfig() != null,
+          "BigtableIO ReadChangeStream is missing required configurations fields.");
+      checkArgument(
+          getBigtableConfig().getProjectId() != null, "Missing required projectId field.");
+      checkArgument(
+          getBigtableConfig().getInstanceId() != null, "Missing required instanceId field.");
+      checkArgument(getTableId() != null, "Missing required tableId field.");
+
+      BigtableConfig bigtableConfig = getBigtableConfig();
+      if (getBigtableConfig().getAppProfileId() == null
+          || getBigtableConfig().getAppProfileId().get().isEmpty()) {
+        bigtableConfig = bigtableConfig.withAppProfileId(StaticValueProvider.of("default"));
+      }
 
       BigtableConfig metadataTableConfig = getMetadataTableBigtableConfig();
+      String metadataTableId = getMetadataTableId();
       if (metadataTableConfig.getProjectId() == null
           || metadataTableConfig.getProjectId().get().isEmpty()) {
-        metadataTableConfig = metadataTableConfig.withProjectId(getBigtableConfig().getProjectId());
+        metadataTableConfig = metadataTableConfig.withProjectId(bigtableConfig.getProjectId());
       }
       if (metadataTableConfig.getInstanceId() == null
           || metadataTableConfig.getInstanceId().get().isEmpty()) {
-        metadataTableConfig =
-            metadataTableConfig.withInstanceId(getBigtableConfig().getInstanceId());
+        metadataTableConfig = metadataTableConfig.withInstanceId(bigtableConfig.getInstanceId());
       }
-      String metadataTableId = getMetadataTableId();
       if (metadataTableId == null || metadataTableId.isEmpty()) {
         metadataTableId = MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME;
       }
       if (metadataTableConfig.getAppProfileId() == null
           || metadataTableConfig.getAppProfileId().get().isEmpty()) {
         metadataTableConfig =
-            metadataTableConfig.withAppProfileId(getBigtableConfig().getAppProfileId());
+            metadataTableConfig.withAppProfileId(bigtableConfig.getAppProfileId());
       }
 
       Instant startTime = getStartTime();
@@ -2004,18 +2047,22 @@ public class BigtableIO {
       if (changeStreamName == null || changeStreamName.isEmpty()) {
         changeStreamName = UniqueIdGenerator.generateRowKeyPrefix();
       }
+      ExistingPipelineOptions existingPipelineOptions = getExistingPipelineOptions();
+      if (existingPipelineOptions == null) {
+        existingPipelineOptions = ExistingPipelineOptions.FAIL_IF_EXISTS;
+      }
 
       ActionFactory actionFactory = new ActionFactory();
       DaoFactory daoFactory =
           new DaoFactory(
-              getBigtableConfig(),
-              metadataTableConfig,
-              getTableId(),
-              metadataTableId,
-              changeStreamName);
+              bigtableConfig, metadataTableConfig, getTableId(), metadataTableId, changeStreamName);
       ChangeStreamMetrics metrics = new ChangeStreamMetrics();
       InitializeDoFn initializeDoFn =
-          new InitializeDoFn(daoFactory, metadataTableConfig.getAppProfileId().get(), startTime);
+          new InitializeDoFn(
+              daoFactory,
+              metadataTableConfig.getAppProfileId().get(),
+              startTime,
+              existingPipelineOptions);
       DetectNewPartitionsDoFn detectNewPartitionsDoFn =
           new DetectNewPartitionsDoFn(getEndTime(), actionFactory, daoFactory, metrics);
       ReadChangeStreamPartitionDoFn readChangeStreamPartitionDoFn =
@@ -2049,7 +2096,7 @@ public class BigtableIO {
       abstract ReadChangeStream.Builder setMetadataTableBigtableConfig(
           BigtableConfig bigtableConfig);
 
-      abstract ReadChangeStream.Builder setMetadataTableId(String tableId);
+      abstract ReadChangeStream.Builder setMetadataTableId(String metadataTableId);
 
       abstract ReadChangeStream.Builder setStartTime(Instant startTime);
 
@@ -2059,6 +2106,9 @@ public class BigtableIO {
 
       abstract ReadChangeStream.Builder setChangeStreamName(String changeStreamName);
 
+      abstract ReadChangeStream.Builder setExistingPipelineOptions(
+          ExistingPipelineOptions existingPipelineOptions);
+
       abstract ReadChangeStream build();
     }
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java
index 54285a5b332..900ddae570e 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelper.java
@@ -45,6 +45,13 @@ public class ByteStringRangeHelper {
     return ByteString.unsignedLexicographicalComparator().compare(e1, e2);
   }
 
+  private static int compareStartEndKey(ByteString start, ByteString end) {
+    if (end.isEmpty()) {
+      return -1;
+    }
+    return ByteString.unsignedLexicographicalComparator().compare(start, end);
+  }
+
   @VisibleForTesting
   static class PartitionComparator implements Comparator<ByteStringRange> {
     @Override
@@ -65,36 +72,37 @@ public class ByteStringRangeHelper {
   }
 
   /**
-   * Returns true if parentPartitions is a superset of childPartition.
+   * Returns true if parentPartitions form a proper superset of childPartition.
    *
-   * <p>If ordered parentPartitions row ranges form a contiguous range, and start key is before or
-   * at childPartition's start key, and end key is at or after childPartition's end key, then
-   * parentPartitions is a superset of childPartition.
+   * <p>If ordered parentPartitions row ranges form a contiguous range without any overlaps, and
+   * start key equals to childPartition's start key, and end key equals to childPartition's end key,
+   * then parentPartitions form a proper superset of childPartition.
    *
-   * <p>Overlaps from parents are valid because arbitrary partitions can merge and they may overlap.
-   * They will form a valid new partition. However, if there are any missing parent partitions, then
-   * merge cannot happen with missing row ranges.
+   * <p>Returns false if there are overlaps or missing partitions in parentPartitions.
    *
-   * @param parentPartitions list of partitions to determine if it forms a large contiguous range
-   * @param childPartition the smaller partition
-   * @return true if parentPartitions is a superset of childPartition, otherwise false.
+   * @param parentPartitions list of partitions to determine if it forms an exact contiguous range
+   * @param childPartition the partition to match
+   * @return true if parentPartitions covers the same key space as childPartition, otherwise false.
    */
-  public static boolean isSuperset(
+  public static boolean coverSameKeySpace(
       List<ByteStringRange> parentPartitions, ByteStringRange childPartition) {
-    // sort parentPartitions by starting key
-    // iterate through, check open end key and close start key of each iteration to ensure no gaps.
-    // first start key and last end key must be equal to or wider than child partition start and end
-    // key.
+    // Sort parentPartitions by starting key. Verify there's no gap or overlaps in between parent
+    // partitions.
+    // The first start key and last end key must be equal to child partition.
     if (parentPartitions.isEmpty()) {
       return false;
     }
     parentPartitions.sort(new PartitionComparator());
-    ByteString parentStartKey = parentPartitions.get(0).getStart();
-    ByteString parentEndKey = parentPartitions.get(parentPartitions.size() - 1).getEnd();
-
-    return !childStartsBeforeParent(parentStartKey, childPartition.getStart())
-        && !childEndsAfterParent(parentEndKey, childPartition.getEnd())
-        && !gapsInParentPartitions(parentPartitions);
+    if (!parentPartitions.get(0).getStart().equals(childPartition.getStart())) {
+      return false;
+    }
+    if (!parentPartitions
+        .get(parentPartitions.size() - 1)
+        .getEnd()
+        .equals(childPartition.getEnd())) {
+      return false;
+    }
+    return isContinuous(parentPartitions);
   }
 
   /**
@@ -110,20 +118,18 @@ public class ByteStringRangeHelper {
   }
 
   /**
-   * Figure out if partitions cover the entire keyspace. If it doesn't, return a list of missing and
-   * overlapping partitions.
+   * Return missing partitions within partitions that are within start and end. Gaps in partitions
+   * that are outside of start and end are ignored.
    *
-   * <p>partitions covers the entire key space if, when ordered, the end key is the same as the
-   * start key of the next row range in the list, and the first start key is "" and the last end key
-   * is "". There should be no overlap.
-   *
-   * @param partitions to determine if they cover entire keyspace
-   * @return list of missing and overlapping partitions
+   * @param partitions find missing partitions within
+   * @param start start of the search range
+   * @param end end of the search range
+   * @return missing partitions within the range
    */
-  public static List<ByteStringRange> getMissingAndOverlappingPartitionsFromKeySpace(
-      List<ByteStringRange> partitions) {
+  public static List<ByteStringRange> getMissingPartitionsFrom(
+      List<ByteStringRange> partitions, ByteString start, ByteString end) {
     if (partitions.isEmpty()) {
-      return Collections.singletonList(ByteStringRange.create("", ""));
+      return Collections.singletonList(ByteStringRange.create(start, end));
     }
 
     List<ByteStringRange> missingPartitions = new ArrayList<>();
@@ -132,22 +138,81 @@ public class ByteStringRangeHelper {
     // iterate through ensuring end key is lexicographically after next start key.
     partitions.sort(new PartitionComparator());
 
-    ByteString prevEnd = ByteString.EMPTY;
-    for (ByteStringRange partition : partitions) {
-      if (!partition.getStart().equals(prevEnd)) {
-        ByteStringRange missingPartition = ByteStringRange.create(prevEnd, partition.getStart());
-        missingPartitions.add(missingPartition);
+    // Check if the start is missing.
+    if (compareStartKey(start, partitions.get(0).getStart()) < 0) {
+      missingPartitions.add(ByteStringRange.create(start, partitions.get(0).getStart()));
+    }
+    ByteString currEnd = partitions.get(0).getEnd();
+    if (compareStartKey(start, partitions.get(0).getEnd()) > 0) {
+      currEnd = start;
+    }
+    for (int i = 1; i < partitions.size(); i++) {
+      if (compareStartEndKey(partitions.get(i).getStart(), currEnd) > 0) {
+        missingPartitions.add(ByteStringRange.create(currEnd, partitions.get(i).getStart()));
+      }
+      // It's possible that we have subset partitions that has a later start key but earlier end
+      // key. So we only advance end key if it's further ahead. For example [A, D) and [B, C).
+      // PrevEnd = D and processing [B, C) would push back end key to C instead of D.
+      if (compareEndKey(currEnd, partitions.get(i).getEnd()) < 0) {
+        currEnd = partitions.get(i).getEnd();
+      }
+      // Current end maybe before the start of the evaluation range.
+      if (compareStartKey(start, currEnd) > 0) {
+        currEnd = start;
+      }
+      // If it's past the end, we stop
+      if (compareEndKey(currEnd, end) > 0) {
+        break;
       }
-      prevEnd = partition.getEnd();
     }
-    // Check that the last partition ends with "", otherwise it's missing.
-    if (!prevEnd.equals(ByteString.EMPTY)) {
-      ByteStringRange missingPartition = ByteStringRange.create(prevEnd, ByteString.EMPTY);
-      missingPartitions.add(missingPartition);
+    // Check that the last partition ends with end, otherwise it's missing.
+    if (compareEndKey(currEnd, end) < 0) {
+      missingPartitions.add(ByteStringRange.create(currEnd, end));
     }
     return missingPartitions;
   }
 
+  /**
+   * Return missing partitions from the entire keyspace.
+   *
+   * @param partitions find missing partitions within
+   * @return a list of missing partitions.
+   */
+  public static List<ByteStringRange> getMissingPartitionsFromEntireKeySpace(
+      List<ByteStringRange> partitions) {
+    return getMissingPartitionsFrom(partitions, ByteString.EMPTY, ByteString.EMPTY);
+  }
+
+  /**
+   * Return a list of overlapping partitions. The partitions outputted are valid partitions i.e.
+   * start < end. This represents 2 ordered partitions overlapping because the 2nd partition starts
+   * before the 1st ended. If end == start, it's not considered overlap.
+   *
+   * @param partitions find overlaps within
+   * @return a list of overlapping partitions.
+   */
+  public static List<ByteStringRange> getOverlappingPartitions(List<ByteStringRange> partitions) {
+    if (partitions.isEmpty() || partitions.size() == 1) {
+      return Collections.emptyList();
+    }
+
+    List<ByteStringRange> overlappingPartitions = new ArrayList<>();
+
+    // sort partitions by start key
+    // iterate through ensuring end key is lexicographically after next start key.
+    partitions.sort(new PartitionComparator());
+
+    // Determine where we start
+    ByteString prevEnd = partitions.get(0).getEnd();
+    for (int i = 1; i < partitions.size(); i++) {
+      if (compareStartEndKey(partitions.get(i).getStart(), prevEnd) < 0) {
+        overlappingPartitions.add(ByteStringRange.create(partitions.get(i).getStart(), prevEnd));
+      }
+      prevEnd = partitions.get(i).getEnd();
+    }
+    return overlappingPartitions;
+  }
+
   /**
    * Returns formatted string of a partition for debugging.
    *
@@ -162,32 +227,18 @@ public class ByteStringRangeHelper {
         + "')";
   }
 
-  private static boolean childStartsBeforeParent(
-      ByteString parentStartKey, ByteString childStartKey) {
-    // Check if the start key of the child partition comes before the start key of the entire
-    // parentPartitions
-    return compareStartKey(parentStartKey, childStartKey) > 0;
-  }
-
-  private static boolean childEndsAfterParent(ByteString parentEndKey, ByteString childEndKey) {
-    return compareEndKey(parentEndKey, childEndKey) < 0;
-  }
-
   // This assumes parentPartitions is sorted. If parentPartitions has not already been sorted
   // it will be incorrect
-  private static boolean gapsInParentPartitions(List<ByteStringRange> sortedParentPartitions) {
+  private static boolean isContinuous(List<ByteStringRange> sortedParentPartitions) {
     for (int i = 1; i < sortedParentPartitions.size(); i++) {
-      // Iterating through a sorted list, the start key should be the same or before the end of the
-      // previous. Handle "" end key as a special case.
+      // Iterating through a sorted list, the start key should be the same as the end of the
+      // previous.
       ByteString prevEndKey = sortedParentPartitions.get(i - 1).getEnd();
-      if (ByteString.unsignedLexicographicalComparator()
-                  .compare(sortedParentPartitions.get(i).getStart(), prevEndKey)
-              > 0
-          && !prevEndKey.isEmpty()) {
-        return true;
+      if (!sortedParentPartitions.get(i).getStart().equals(prevEndKey)) {
+        return false;
       }
     }
-    return false;
+    return true;
   }
 
   /**
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ChangeStreamMetrics.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ChangeStreamMetrics.java
index b3548961f43..f732a52fd37 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ChangeStreamMetrics.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ChangeStreamMetrics.java
@@ -34,27 +34,33 @@ public class ChangeStreamMetrics implements Serializable {
    * Counter for the total number of partitions identified during the execution of the Connector.
    */
   public static final Counter LIST_PARTITIONS_COUNT =
-      Metrics.counter(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "list_partitions_count");
+      Metrics.counter(ChangeStreamMetrics.class, "list_partitions_count");
 
   /**
    * Counter for the total number of partition splits / moves identified during the execution of the
    * Connector.
    */
   public static final Counter PARTITION_SPLIT_COUNT =
-      Metrics.counter(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "partition_record_split_count");
+      Metrics.counter(ChangeStreamMetrics.class, "partition_record_split_count");
 
   /**
    * Counter for the total number of partition merges identified during the execution of the
    * Connector.
    */
   public static final Counter PARTITION_MERGE_COUNT =
-      Metrics.counter(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "partition_record_merge_count");
+      Metrics.counter(ChangeStreamMetrics.class, "partition_record_merge_count");
+
+  /** Counter for the total number of partitions reconciled with continuation tokens. */
+  public static final Counter PARTITION_RECONCILED_WITH_TOKEN_COUNT =
+      Metrics.counter(ChangeStreamMetrics.class, "partition_reconciled_with_token_count");
+
+  /** Counter for the total number of partitions reconciled without continuation tokens. */
+  public static final Counter PARTITION_RECONCILED_WITHOUT_TOKEN_COUNT =
+      Metrics.counter(ChangeStreamMetrics.class, "partition_reconciled_without_token_count");
+
+  /** Counter for the total number of orphaned new partitions cleaned up. */
+  public static final Counter ORPHANED_NEW_PARTITION_CLEANED_COUNT =
+      Metrics.counter(ChangeStreamMetrics.class, "orphaned_new_partition_cleaned_count");
 
   // -------------------
   // Read change stream metrics
@@ -63,132 +69,111 @@ public class ChangeStreamMetrics implements Serializable {
    * Counter for the total number of heartbeats identified during the execution of the Connector.
    */
   public static final Counter HEARTBEAT_COUNT =
-      Metrics.counter(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "heartbeat_count");
+      Metrics.counter(ChangeStreamMetrics.class, "heartbeat_count");
 
   /**
    * Counter for the total number of heartbeats identified during the execution of the Connector.
    */
   public static final Counter CLOSESTREAM_COUNT =
-      Metrics.counter(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "closestream_count");
+      Metrics.counter(ChangeStreamMetrics.class, "closestream_count");
 
   /**
    * Counter for the total number of ChangeStreamMutations that are initiated by users (not garbage
    * collection) identified during the execution of the Connector.
    */
   public static final Counter CHANGE_STREAM_MUTATION_USER_COUNT =
-      Metrics.counter(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "change_stream_mutation_user_count");
+      Metrics.counter(ChangeStreamMetrics.class, "change_stream_mutation_user_count");
 
   /**
    * Counter for the total number of ChangeStreamMutations that are initiated by garbage collection
    * (not user initiated) identified during the execution of the Connector.
    */
   public static final Counter CHANGE_STREAM_MUTATION_GC_COUNT =
-      Metrics.counter(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "change_stream_mutation_gc_count");
+      Metrics.counter(ChangeStreamMetrics.class, "change_stream_mutation_gc_count");
 
   /** Distribution for measuring processing delay from commit timestamp. */
   public static final Distribution PROCESSING_DELAY_FROM_COMMIT_TIMESTAMP =
-      Metrics.distribution(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "processing_delay_from_commit_timestamp");
+      Metrics.distribution(ChangeStreamMetrics.class, "processing_delay_from_commit_timestamp");
 
   /** Counter for the total number of active partitions being streamed. */
   public static final Counter PARTITION_STREAM_COUNT =
-      Metrics.counter(
-          org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics.class,
-          "partition_stream_count");
+      Metrics.counter(ChangeStreamMetrics.class, "partition_stream_count");
 
   /**
-   * Increments the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#LIST_PARTITIONS_COUNT} by
-   * 1 if the metric is enabled.
+   * Increments the {@link ChangeStreamMetrics#LIST_PARTITIONS_COUNT} by 1 if the metric is enabled.
    */
   public void incListPartitionsCount() {
     inc(LIST_PARTITIONS_COUNT);
   }
 
   /**
-   * Increments the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PARTITION_SPLIT_COUNT} by
-   * 1 if the metric is enabled.
+   * Increments the {@link ChangeStreamMetrics#PARTITION_SPLIT_COUNT} by 1 if the metric is enabled.
    */
   public void incPartitionSplitCount() {
     inc(PARTITION_SPLIT_COUNT);
   }
 
   /**
-   * Increments the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PARTITION_MERGE_COUNT} by
-   * 1 if the metric is enabled.
+   * Increments the {@link ChangeStreamMetrics#PARTITION_MERGE_COUNT} by 1 if the metric is enabled.
    */
   public void incPartitionMergeCount() {
     inc(PARTITION_MERGE_COUNT);
   }
 
-  /**
-   * Increments the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#HEARTBEAT_COUNT} by 1 if
-   * the metric is enabled.
-   */
+  /** Increments the {@link ChangeStreamMetrics#PARTITION_RECONCILED_WITH_TOKEN_COUNT} by 1. */
+  public void incPartitionReconciledWithTokenCount() {
+    inc(PARTITION_RECONCILED_WITH_TOKEN_COUNT);
+  }
+
+  /** Increments the {@link ChangeStreamMetrics#PARTITION_RECONCILED_WITHOUT_TOKEN_COUNT} by 1. */
+  public void incPartitionReconciledWithoutTokenCount() {
+    inc(PARTITION_RECONCILED_WITHOUT_TOKEN_COUNT);
+  }
+
+  /** Increments the {@link ChangeStreamMetrics#ORPHANED_NEW_PARTITION_CLEANED_COUNT} by 1. */
+  public void incOrphanedNewPartitionCleanedCount() {
+    inc(ORPHANED_NEW_PARTITION_CLEANED_COUNT);
+  }
+
+  /** Increments the {@link ChangeStreamMetrics#HEARTBEAT_COUNT} by 1 if the metric is enabled. */
   public void incHeartbeatCount() {
     inc(HEARTBEAT_COUNT);
   }
 
-  /**
-   * Increments the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#CLOSESTREAM_COUNT} by 1
-   * if the metric is enabled.
-   */
+  /** Increments the {@link ChangeStreamMetrics#CLOSESTREAM_COUNT} by 1 if the metric is enabled. */
   public void incClosestreamCount() {
     inc(CLOSESTREAM_COUNT);
   }
 
   /**
-   * Increments the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#CHANGE_STREAM_MUTATION_USER_COUNT}
-   * by 1 if the metric is enabled.
+   * Increments the {@link ChangeStreamMetrics#CHANGE_STREAM_MUTATION_USER_COUNT} by 1 if the metric
+   * is enabled.
    */
   public void incChangeStreamMutationUserCounter() {
     inc(CHANGE_STREAM_MUTATION_USER_COUNT);
   }
 
   /**
-   * Increments the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#CHANGE_STREAM_MUTATION_GC_COUNT}
-   * by 1 if the metric is enabled.
+   * Increments the {@link ChangeStreamMetrics#CHANGE_STREAM_MUTATION_GC_COUNT} by 1 if the metric
+   * is enabled.
    */
   public void incChangeStreamMutationGcCounter() {
     inc(CHANGE_STREAM_MUTATION_GC_COUNT);
   }
 
-  /**
-   * Increments the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PARTITION_STREAM_COUNT}
-   * by 1.
-   */
+  /** Increments the {@link ChangeStreamMetrics#PARTITION_STREAM_COUNT} by 1. */
   public void incPartitionStreamCount() {
     inc(PARTITION_STREAM_COUNT);
   }
 
-  /**
-   * Decrements the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PARTITION_STREAM_COUNT}
-   * by 1.
-   */
+  /** Decrements the {@link ChangeStreamMetrics#PARTITION_STREAM_COUNT} by 1. */
   public void decPartitionStreamCount() {
     dec(PARTITION_STREAM_COUNT);
   }
 
   /**
    * Adds measurement of an instance for the {@link
-   * org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics#PROCESSING_DELAY_FROM_COMMIT_TIMESTAMP}.
+   * ChangeStreamMetrics#PROCESSING_DELAY_FROM_COMMIT_TIMESTAMP}.
    */
   public void updateProcessingDelayFromCommitTimestamp(long durationInMilli) {
     update(PROCESSING_DELAY_FROM_COMMIT_TIMESTAMP, durationInMilli);
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverter.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverter.java
index 4b664943f6a..bc58ef781ad 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverter.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverter.java
@@ -34,4 +34,8 @@ public class TimestampConverter {
   public static long toSeconds(org.joda.time.Instant jodaInstant) {
     return jodaInstant.getMillis() / 1000;
   }
+
+  public static org.joda.time.Instant microsecondToInstant(long microsecond) {
+    return org.joda.time.Instant.ofEpochMilli(microsecond / 1_000L);
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/UniqueIdGenerator.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/UniqueIdGenerator.java
index c5677f3feae..6999f47b6f3 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/UniqueIdGenerator.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/UniqueIdGenerator.java
@@ -26,14 +26,29 @@ import org.apache.beam.sdk.annotations.Internal;
 public class UniqueIdGenerator {
   private static final SecureRandom secureRandom = new SecureRandom();
 
+  /**
+   * Return a random base64 encoded 8 byte string. This is used to identify streamer of a specific
+   * partition. We expect there to be at most single digit duplicates of the same partition at any
+   * one time. The odd of collision if we generate <10 values per partition is less than 1 in 10^18.
+   *
+   * @return a random 8 byte string.
+   */
   public static String getNextId() {
-    byte[] bytes = new byte[256];
+    byte[] bytes = new byte[8];
     secureRandom.nextBytes(bytes);
     return Base64.getEncoder().encodeToString(bytes);
   }
 
+  /**
+   * Return a random base64 encoded 8 byte string. This is used to identify a pipeline. Once a
+   * pipeline is complete, the metadata is left there and not cleaned up. It's possible, over the
+   * lifetime, there to be thousands and more pipelines in a single metadata table. The odds of
+   * collision if we expect 100,000 pipelines is less than 1 in 1 billion.
+   *
+   * @return a random 8 byte string.
+   */
   public static String generateRowKeyPrefix() {
-    byte[] bytes = new byte[50];
+    byte[] bytes = new byte[8];
     secureRandom.nextBytes(bytes);
     return Base64.getEncoder().encodeToString(bytes);
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java
index ac586ea147d..43623b5de16 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ActionFactory.java
@@ -42,7 +42,9 @@ public class ActionFactory implements Serializable {
 
   private transient ChangeStreamAction changeStreamAction;
   private transient DetectNewPartitionsAction detectNewPartitionsAction;
+  private transient ProcessNewPartitionsAction processNewPartitionsAction;
   private transient GenerateInitialPartitionsAction generateInitialPartitionsAction;
+  private transient ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction;
   private transient ReadChangeStreamPartitionAction readChangeStreamPartitionAction;
 
   /**
@@ -75,15 +77,31 @@ public class ActionFactory implements Serializable {
       ChangeStreamMetrics metrics,
       MetadataTableDao metadataTableDao,
       @Nullable Instant endTime,
-      GenerateInitialPartitionsAction generateInitialPartitionsAction) {
+      GenerateInitialPartitionsAction generateInitialPartitionsAction,
+      ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction,
+      ProcessNewPartitionsAction processNewPartitionsAction) {
     if (detectNewPartitionsAction == null) {
       detectNewPartitionsAction =
           new DetectNewPartitionsAction(
-              metrics, metadataTableDao, endTime, generateInitialPartitionsAction);
+              metrics,
+              metadataTableDao,
+              endTime,
+              generateInitialPartitionsAction,
+              resumeFromPreviousPipelineAction,
+              processNewPartitionsAction);
     }
     return detectNewPartitionsAction;
   }
 
+  public synchronized ProcessNewPartitionsAction processNewPartitionsAction(
+      ChangeStreamMetrics metrics, MetadataTableDao metadataTableDao, @Nullable Instant endTime) {
+    if (processNewPartitionsAction == null) {
+      processNewPartitionsAction =
+          new ProcessNewPartitionsAction(metrics, metadataTableDao, endTime);
+    }
+    return processNewPartitionsAction;
+  }
+
   /**
    * Creates and returns a singleton instance of an action class for processing {@link
    * org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.DetectNewPartitionsDoFn}
@@ -101,6 +119,19 @@ public class ActionFactory implements Serializable {
     return generateInitialPartitionsAction;
   }
 
+  public synchronized ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction(
+      ChangeStreamMetrics metrics,
+      MetadataTableDao metadataTableDao,
+      @Nullable Instant endTime,
+      ProcessNewPartitionsAction processNewPartitionsAction) {
+    if (resumeFromPreviousPipelineAction == null) {
+      resumeFromPreviousPipelineAction =
+          new ResumeFromPreviousPipelineAction(
+              metrics, metadataTableDao, endTime, processNewPartitionsAction);
+    }
+    return resumeFromPreviousPipelineAction;
+  }
+
   /**
    * Creates and returns a singleton instance of an action class for processing {@link
    * org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.ReadChangeStreamPartitionDoFn}.
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java
index 7c002b8ede9..4c3f9c5a833 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamAction.java
@@ -136,7 +136,7 @@ public class ChangeStreamAction {
       if (!tracker.tryClaim(streamProgress)) {
         if (shouldDebug) {
           LOG.info(
-              "RCSP {}: Failed to claim heart beat tracker",
+              "RCSP {}: Checkpoint heart beat tracker",
               formatByteStringRange(partitionRecord.getPartition()));
         }
         return Optional.of(DoFn.ProcessContinuation.stop());
@@ -167,7 +167,7 @@ public class ChangeStreamAction {
       if (!tracker.tryClaim(streamProgress)) {
         if (shouldDebug) {
           LOG.info(
-              "RCSP {}: Failed to claim close stream tracker",
+              "RCSP {}: Checkpoint close stream tracker",
               formatByteStringRange(partitionRecord.getPartition()));
         }
         return Optional.of(DoFn.ProcessContinuation.stop());
@@ -192,7 +192,7 @@ public class ChangeStreamAction {
       if (!tracker.tryClaim(streamProgress)) {
         if (shouldDebug) {
           LOG.info(
-              "RCSP {}: Failed to claim data change tracker",
+              "RCSP {}: Checkpoint data change tracker",
               formatByteStringRange(partitionRecord.getPartition()));
         }
         return Optional.of(DoFn.ProcessContinuation.stop());
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java
index 07ca7775436..d444f6a2dfd 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsAction.java
@@ -17,42 +17,35 @@
  */
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
 
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingAndOverlappingPartitionsFromKeySpace;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.isSuperset;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingPartitionsFromEntireKeySpace;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getOverlappingPartitions;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.partitionsToString;
 
-import com.google.api.gax.rpc.ServerStream;
-import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
 import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
-import com.google.cloud.bigtable.data.v2.models.Row;
-import com.google.cloud.bigtable.data.v2.models.RowCell;
-import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
 import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Set;
+import java.util.Optional;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Internal;
-import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
-import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
-import org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.DetectNewPartitionsState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.StreamPartitionWithWatermark;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler.OrphanedMetadataCleaner;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler.PartitionReconciler;
 import org.apache.beam.sdk.io.range.OffsetRange;
-import org.apache.beam.sdk.transforms.DoFn.BundleFinalizer;
 import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
 import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
 import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.slf4j.Logger;
@@ -74,26 +67,30 @@ public class DetectNewPartitionsAction {
   private final ChangeStreamMetrics metrics;
   private final MetadataTableDao metadataTableDao;
   @Nullable private final Instant endTime;
+  private final ProcessNewPartitionsAction processNewPartitionsAction;
   private final GenerateInitialPartitionsAction generateInitialPartitionsAction;
+  private final ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction;
 
   private transient PartitionReconciler partitionReconciler;
+  private transient OrphanedMetadataCleaner orphanedMetadataCleaner;
 
   public DetectNewPartitionsAction(
       ChangeStreamMetrics metrics,
       MetadataTableDao metadataTableDao,
       @Nullable Instant endTime,
-      GenerateInitialPartitionsAction generateInitialPartitionsAction) {
+      GenerateInitialPartitionsAction generateInitialPartitionsAction,
+      ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction,
+      ProcessNewPartitionsAction processNewPartitionsAction) {
     this.metrics = metrics;
     this.metadataTableDao = metadataTableDao;
     this.endTime = endTime;
     this.generateInitialPartitionsAction = generateInitialPartitionsAction;
+    this.resumeFromPreviousPipelineAction = resumeFromPreviousPipelineAction;
+    this.processNewPartitionsAction = processNewPartitionsAction;
   }
 
   /**
-   * Periodically advances DetectNewPartition's (DNP) watermark based on the watermark of all the
-   * partitions recorded in the metadata table. We don't advance DNP's watermark on every run to
-   * "now" because of a possible inconsistent state. DNP's watermark is used to hold the entire
-   * pipeline's watermark back.
+   * Get the new watermark based on the watermark of StreamPartitions and NewPartitions.
    *
    * <p>The low watermark is important to determine when to terminate the pipeline. During splits
    * and merges, the watermark step may appear to be higher than it actually is. If a partition, at
@@ -110,171 +107,65 @@ public class DetectNewPartitionsAction {
    * ensuring all the partitions exists in the metadata table in order to calculate the low
    * watermark. It is possible that some partitions might be missing in between split and merges.
    *
-   * @param tracker restriction tracker to guide how frequently watermark should be advanced
-   * @param watermarkEstimator watermark estimator to advance the watermark
+   * @param streamPartitionsWithWatermark restriction tracker to guide how frequently watermark
+   *     should be advanced
+   * @param newPartitions watermark estimator to advance the watermark
+   * @return low watermark if possible, otherwise empty.
    */
-  private void advanceWatermark(
-      RestrictionTracker<OffsetRange, Long> tracker,
-      ManualWatermarkEstimator<Instant> watermarkEstimator)
-      throws InvalidProtocolBufferException {
-    // We currently choose to update the watermark every 10 runs. We want to choose a number that is
-    // frequent, so the watermark isn't lagged behind too far. Also not too frequent so we do not
-    // overload the table with full table scans.
-    if (tracker.currentRestriction().getFrom() % 10 == 0) {
-      // Get partitions with a watermark set but skip rows w a lock and no watermark yet
-      ServerStream<Row> rows = metadataTableDao.readFromMdTableStreamPartitionsWithWatermark();
-      List<ByteStringRange> partitions = new ArrayList<>();
-      HashMap<ByteStringRange, Instant> slowPartitions = new HashMap<>();
-      Instant lowWatermark = Instant.ofEpochMilli(Long.MAX_VALUE);
-      for (Row row : rows) {
-        Instant watermark = MetadataTableEncoder.parseWatermarkFromRow(row);
-        if (watermark == null) {
-          continue;
-        }
-        // Update low watermark if watermark < low watermark.
-        if (watermark.compareTo(lowWatermark) < 0) {
-          lowWatermark = watermark;
-        }
-        ByteStringRange partition =
-            metadataTableDao.convertStreamPartitionRowKeyToPartition(row.getKey());
-        partitions.add(partition);
-        if (watermark.plus(DEBUG_WATERMARK_DELAY).isBeforeNow()) {
-          slowPartitions.put(partition, watermark);
-        }
-      }
-      List<ByteStringRange> missingAndOverlappingPartitions =
-          getMissingAndOverlappingPartitionsFromKeySpace(partitions);
-      if (missingAndOverlappingPartitions.isEmpty()) {
-        watermarkEstimator.setWatermark(lowWatermark);
-        LOG.info("DNP: Updating watermark: " + watermarkEstimator.currentWatermark());
-      } else {
-        LOG.warn(
-            "DNP: Could not update watermark because missing {}",
-            partitionsToString(missingAndOverlappingPartitions));
+  private Optional<Instant> getNewWatermark(
+      List<StreamPartitionWithWatermark> streamPartitionsWithWatermark,
+      List<NewPartition> newPartitions) {
+    // Get partitions with a watermark set but skip rows w a lock and no watermark yet
+    List<StreamPartitionWithWatermark> slowPartitions = new ArrayList<>();
+    Instant lowWatermark = Instant.ofEpochMilli(Long.MAX_VALUE);
+
+    List<ByteStringRange> partitions = new ArrayList<>();
+    for (StreamPartitionWithWatermark streamPartitionWithWatermark :
+        streamPartitionsWithWatermark) {
+      if (streamPartitionWithWatermark.getWatermark().plus(DEBUG_WATERMARK_DELAY).isBeforeNow()) {
+        slowPartitions.add(streamPartitionWithWatermark);
       }
-      if (!slowPartitions.isEmpty()) {
-        LOG.warn(
-            "DNP: Watermark is being held back by the following partitions: {}",
-            slowPartitions.entrySet().stream()
-                .map(
-                    e ->
-                        ByteStringRangeHelper.formatByteStringRange(e.getKey())
-                            + " => "
-                            + e.getValue())
-                .collect(Collectors.joining(", ", "{", "}")));
+      if (streamPartitionWithWatermark.getWatermark().compareTo(lowWatermark) < 0) {
+        lowWatermark = streamPartitionWithWatermark.getWatermark();
       }
-      partitionReconciler.addMissingPartitions(missingAndOverlappingPartitions);
-    }
-  }
-
-  /**
-   * Process a single new partition. New partition resulting from split and merges need to be
-   * outputted to be streamed. Regardless if it's a split or a merge, we have the same verification
-   * process in order to ensure the new partition can actually be streamed.
-   *
-   * <p>When a parent partition splits, it receives two or more new partitions. It will write a new
-   * row, with the new row ranges as row key, for each new partition. These new partitions can be
-   * immediately streamed.
-   *
-   * <p>The complicated scenario is merges. Two or more parent partitions will merge into one new
-   * partition. Each parent partition receives the same new partition (row range) but each parent
-   * partition will have a different continuation token. The parent partitions will all write to the
-   * same row key form by the new row range. Each parent will record its continuation token, and
-   * watermark. Parent partitions may not receive the message to stop at the same time. So when we
-   * try to process the new partition, we need to ensure that all the parent partitions have stopped
-   * and recorded their metadata table. We do so by verifying that the row ranges of the parents
-   * covers a contiguous block of row range that is same or wider (superset) of the new row range.
-   *
-   * <p>For example, partition1, A-B, and partition2, B-C, merges into partition3, A-C.
-   *
-   * <ol>
-   *   <li>p1 writes to row A-C in metadata table
-   *   <li>processNewPartition process A-C seeing that only A-B has been recorded and A-B does not
-   *       cover A-C. Do Nothing
-   *   <li>p2 writes to row A-C in metadata table
-   *   <li>processNewPartition process A-C again, seeing that A-B and B-C has been recorded and
-   *       outputs new partition A-C to be streamed.
-   * </ol>
-   *
-   * <p>Note that, the algorithm to verify if a merge is valid, also correctly verifies if a split
-   * is valid. A split is immediately valid as long as the row exists because there's only one
-   * parent that needs to write to that row.
-   *
-   * @param row new partition to be processed
-   * @param receiver to output new partitions
-   * @param recordedPartitionRecords add the partition if it's been processed
-   * @throws InvalidProtocolBufferException if partition can't be converted from row key.
-   */
-  private void processNewPartition(
-      Row row, OutputReceiver<PartitionRecord> receiver, List<ByteString> recordedPartitionRecords)
-      throws Exception {
-    ByteStringRange partition = metadataTableDao.convertNewPartitionRowKeyToPartition(row.getKey());
-
-    partitionReconciler.addNewPartition(partition, row.getKey());
-
-    // Ensure all parent partitions have stopped and updated the metadata table with the new
-    // partition continuation token.
-    List<ByteStringRange> parentPartitions = new ArrayList<>();
-    for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_PARENT_PARTITIONS)) {
-      ByteStringRange parentPartition = ByteStringRange.toByteStringRange(cell.getQualifier());
-      parentPartitions.add(parentPartition);
+      partitions.add(streamPartitionWithWatermark.getPartition());
     }
-    if (!isSuperset(parentPartitions, partition)) {
+    if (!slowPartitions.isEmpty()) {
       LOG.warn(
-          "DNP: New partition: {} does not have all the parents {}",
-          ByteStringRangeHelper.formatByteStringRange(partition),
-          partitionsToString(parentPartitions));
-      return;
-    }
-
-    // Capture all the initial continuation tokens to request for the new partition.
-    ImmutableList.Builder<ChangeStreamContinuationToken> changeStreamContinuationTokensBuilder =
-        ImmutableList.builder();
-    for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_INITIAL_TOKEN)) {
-      ChangeStreamContinuationToken changeStreamContinuationToken =
-          ChangeStreamContinuationToken.fromByteString(cell.getQualifier());
-      changeStreamContinuationTokensBuilder.add(changeStreamContinuationToken);
+          "DNP: Updating watermark is held back by {} partitions : {}",
+          slowPartitions.size(),
+          slowPartitions.stream()
+              .map(e -> formatByteStringRange(e.getPartition()) + " => " + e.getWatermark())
+              .collect(Collectors.joining(", ", "{", "}")));
     }
-    ImmutableList<ChangeStreamContinuationToken> changeStreamContinuationTokens =
-        changeStreamContinuationTokensBuilder.build();
 
-    // If parent and continuation token count are not the same, it's possible the same parent wrote
-    // multiple continuation tokens or there's an inconsistent state.
-    if (parentPartitions.size() != changeStreamContinuationTokens.size()) {
+    // Only added StreamPartitions so far, check if there are any overlapping StreamPartitions. If
+    // so, something is wrong, we should stop.
+    List<ByteStringRange> overlappingStreamPartitions = getOverlappingPartitions(partitions);
+    if (!overlappingStreamPartitions.isEmpty()) {
       LOG.warn(
-          "DNP: New partition {} parent partitions count {} != continuation token count {}",
-          ByteStringRangeHelper.formatByteStringRange(partition),
-          parentPartitions.size(),
-          changeStreamContinuationTokens.size());
+          "DNP: Updating watermark failed due to overlapping: {}",
+          partitionsToString(overlappingStreamPartitions));
+      return Optional.empty();
     }
 
-    // Capture all the low watermark and calculate the low watermark among all the parents.
-    List<Long> parentLowWatermark = new ArrayList<>();
-    for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS)) {
-      parentLowWatermark.add(Longs.fromByteArray(cell.getValue().toByteArray()));
+    for (NewPartition newPartition : newPartitions) {
+      partitions.addAll(newPartition.getParentPartitions());
+      if (newPartition.getLowWatermark().compareTo(lowWatermark) < 0) {
+        lowWatermark = newPartition.getLowWatermark();
+      }
     }
-    Long lowWatermarkMilli = Collections.min(parentLowWatermark);
-    Instant lowWatermark = Instant.ofEpochMilli(lowWatermarkMilli);
 
-    String uid = UniqueIdGenerator.getNextId();
-    PartitionRecord partitionRecord =
-        new PartitionRecord(partition, changeStreamContinuationTokens, uid, lowWatermark, endTime);
-    if (parentPartitions.size() > 1) {
-      metrics.incPartitionMergeCount();
-    } else {
-      metrics.incPartitionSplitCount();
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    if (missingPartitions.isEmpty()) {
+      LOG.info("DNP: Updating watermark: " + lowWatermark);
+      return Optional.of(lowWatermark);
     }
-    LOG.info(
-        "DNP: Split/Merge {} into {}",
-        parentPartitions.stream()
-            .map(ByteStringRangeHelper::formatByteStringRange)
-            .collect(Collectors.joining(",", "{", "}")),
-        ByteStringRangeHelper.formatByteStringRange(partition));
-    // We are outputting elements with timestamp of 0 to prevent reliance on event time. This limits
-    // the ability to window on commit time of any data changes. It is still possible to window on
-    // processing time.
-    receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
-    recordedPartitionRecords.add(row.getKey());
+    LOG.warn(
+        "DNP: Updating watermark failed due to missing {} partitions : {}.",
+        missingPartitions.size(),
+        partitionsToString(missingPartitions));
+    return Optional.empty();
   }
 
   /**
@@ -283,69 +174,54 @@ public class DetectNewPartitionsAction {
    *
    * @param receiver used to output reconciled partitions
    * @param watermarkEstimator read the low watermark for all partitions
-   * @param startTime startTime of the pipeline
-   * @param recordedPartitionRecords the list of row keys that needs to be cleaned up
    */
   private void processReconcilerPartitions(
       OutputReceiver<PartitionRecord> receiver,
       ManualWatermarkEstimator<Instant> watermarkEstimator,
-      Instant startTime,
-      List<ByteString> recordedPartitionRecords) {
-    for (HashMap.Entry<ByteStringRange, Set<ByteString>> partitionToReconcile :
-        partitionReconciler.getPartitionsToReconcile().entrySet()) {
-      String uid = UniqueIdGenerator.getNextId();
+      Instant startTime) {
+    for (PartitionRecord reconciledPartition :
+        partitionReconciler.getPartitionsToReconcile(
+            watermarkEstimator.currentWatermark(), startTime)) {
+      // Set the missing fields.
+      reconciledPartition.setUuid(UniqueIdGenerator.getNextId());
+      reconciledPartition.setEndTime(endTime);
 
-      // When we reconcile, we start from 1h prior to effectively eliminate the possibility of
-      // missing data.
-      Instant reconciledTime =
-          watermarkEstimator.currentWatermark().minus(Duration.standardMinutes(60));
-      if (reconciledTime.compareTo(startTime) < 0) {
-        reconciledTime = startTime;
+      for (NewPartition parentPartition : reconciledPartition.getParentPartitions()) {
+        metadataTableDao.markNewPartitionForDeletion(parentPartition);
       }
+      receiver.outputWithTimestamp(reconciledPartition, Instant.EPOCH);
+      LOG.warn("DNP: Reconciling missing partition: {}", reconciledPartition);
+    }
+  }
 
-      PartitionRecord partitionRecord =
-          new PartitionRecord(
-              partitionToReconcile.getKey(), reconciledTime, uid, reconciledTime, endTime);
-      receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
-      recordedPartitionRecords.addAll(partitionToReconcile.getValue());
-      LOG.warn(
-          "DNP: Reconciling missing partition: {} and cleaning up rows {}",
-          partitionRecord,
-          partitionToReconcile.getValue().stream()
-              .map(
-                  rowKey -> {
-                    try {
-                      return ByteStringRangeHelper.formatByteStringRange(
-                          metadataTableDao.convertNewPartitionRowKeyToPartition(rowKey));
-                    } catch (InvalidProtocolBufferException exception) {
-                      return rowKey.toStringUtf8();
-                    }
-                  })
-              .collect(Collectors.joining(", ", "{", "}")));
+  private void cleanUpOrphanedMetadata() {
+    for (NewPartition newPartitionToClean : orphanedMetadataCleaner.getOrphanedNewPartitions()) {
+      metrics.incOrphanedNewPartitionCleanedCount();
+      metadataTableDao.markNewPartitionForDeletion(newPartitionToClean);
+      metadataTableDao.deleteNewPartition(newPartitionToClean);
     }
   }
 
   /**
-   * After processing new partitions and if it was outputted successfully, we need to clean up the
-   * metadata table so that we don't try to process the same new partition again.
+   * Return true if watermark should be updated.
+   *
+   * <p>We update the watermark on even count after 10s since last update.
    *
-   * <p>This is performed at a best effort basis. If some clean up is unsuccessful, the new
-   * partition will be processed and output again in subsequent runs. This is OK because the DoFn
-   * that stream partitions performs locking to prevent multiple DoFn from streaming the same
-   * partition.
+   * <p>We chose to only update on even count, so we don't perform a full table scan on every DNP
+   * run if DNP taking longer than 10s between each run.
    *
-   * @param bundleFinalizer finalizer that registered the callback to perform the cleanup
-   * @param recordedPartitionRecords the list of row keys that needs to be cleaned up
+   * <p>We set a minimum of 10s to minimize frequency of full table scan. This means the watermark
+   * will always lag behind by 10s. We didn't choose a larger minimum frequency because that would
+   * increase the minimum watermark delay.
    */
-  private void cleanUpAfterCommit(
-      BundleFinalizer bundleFinalizer, List<ByteString> recordedPartitionRecords) {
-    bundleFinalizer.afterBundleCommit(
-        Instant.ofEpochMilli(Long.MAX_VALUE),
-        () -> {
-          for (ByteString rowKey : recordedPartitionRecords) {
-            metadataTableDao.deleteRowKey(rowKey);
-          }
-        });
+  private boolean shouldUpdateWatermark(
+      long count, @Nullable DetectNewPartitionsState detectNewPartitionsState) {
+    return count % 2 == 0
+        && (detectNewPartitionsState == null
+            || detectNewPartitionsState
+                .getWatermarkLastUpdated()
+                .plus(Duration.standardSeconds(10))
+                .isBeforeNow());
   }
 
   /**
@@ -366,7 +242,6 @@ public class DetectNewPartitionsAction {
    * @param receiver output new partitions
    * @param watermarkEstimator update watermark that is a representation of the low watermark of the
    *     entire beam pipeline
-   * @param bundleFinalizer perform after bundle output actions to clean up metadata table
    * @return {@link ProcessContinuation#resume()} with 1-second delay if the stream continues,
    *     otherwise {@link ProcessContinuation#stop()}
    * @throws InvalidProtocolBufferException if failing to process new partitions
@@ -376,23 +251,48 @@ public class DetectNewPartitionsAction {
       RestrictionTracker<OffsetRange, Long> tracker,
       OutputReceiver<PartitionRecord> receiver,
       ManualWatermarkEstimator<Instant> watermarkEstimator,
-      BundleFinalizer bundleFinalizer,
-      Instant startTime)
+      InitialPipelineState initialPipelineState)
       throws Exception {
     LOG.debug("DNP: Watermark: " + watermarkEstimator.getState());
     LOG.debug("DNP: CurrentTracker: " + tracker.currentRestriction().getFrom());
     if (tracker.currentRestriction().getFrom() == 0L) {
-      return generateInitialPartitionsAction.run(receiver, tracker, watermarkEstimator, startTime);
+      if (!tracker.tryClaim(0L)) {
+        LOG.error(
+            "Could not claim initial DetectNewPartition restriction. No partitions are outputted.");
+        return ProcessContinuation.stop();
+      }
+      watermarkEstimator.setWatermark(initialPipelineState.getStartTime());
+      if (initialPipelineState.isResume()) {
+        resumeFromPreviousPipelineAction.run(receiver);
+      } else {
+        generateInitialPartitionsAction.run(receiver, initialPipelineState.getStartTime());
+      }
+      return ProcessContinuation.resume();
     }
 
     // Create a new partition reconciler every run to reset the state each time.
-    partitionReconciler = new PartitionReconciler(metadataTableDao);
+    partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
+    orphanedMetadataCleaner = new OrphanedMetadataCleaner();
 
-    advanceWatermark(tracker, watermarkEstimator);
+    // Calculating the new value of watermark is a resource intensive process. We have to do a full
+    // scan of the metadata table and then ensure we're not missing partitions and then calculate
+    // the low watermark. This is usually a fairly fast process even with thousands of partitions.
+    // However, sometimes this may take so long that the runner checkpoints before the watermark is
+    // calculated. Because the checkpoint takes place before tryClaim, this forces the DoFn to
+    // restart, wasting the resources spent calculating the watermark. On restart, we will try to
+    // calculate the watermark again. The problem causing the slow watermark calculation can persist
+    // leading to a crash loop. In order to ensure we persist the calculated watermark, we calculate
+    // the watermark after successful tryClaim. Then we write to the metadata table the new
+    // watermark. On the start of each run we read the watermark and update the DoFn's watermark.
+    DetectNewPartitionsState detectNewPartitionsState =
+        metadataTableDao.readDetectNewPartitionsState();
+    if (detectNewPartitionsState != null) {
+      watermarkEstimator.setWatermark(detectNewPartitionsState.getWatermark());
+    }
 
     // Terminate if endTime <= watermark that means all partitions have read up to or beyond
     // watermark. We no longer need to manage splits and merges, we can terminate.
-    if (endTime != null && endTime.isBefore(watermarkEstimator.currentWatermark())) {
+    if (endTime != null && !watermarkEstimator.currentWatermark().isBefore(endTime)) {
       tracker.tryClaim(tracker.currentRestriction().getTo());
       return ProcessContinuation.stop();
     }
@@ -402,17 +302,51 @@ public class DetectNewPartitionsAction {
       return ProcessContinuation.stop();
     }
 
-    List<ByteString> recordedPartitionRecords = new ArrayList<>();
+    // Read StreamPartitions to calculate watermark.
+    List<StreamPartitionWithWatermark> streamPartitionsWithWatermark = null;
+    if (shouldUpdateWatermark(tracker.currentRestriction().getFrom(), detectNewPartitionsState)) {
+      streamPartitionsWithWatermark = metadataTableDao.readStreamPartitionsWithWatermark();
+    }
 
-    ServerStream<Row> rows = metadataTableDao.readNewPartitions();
-    for (Row row : rows) {
-      processNewPartition(row, receiver, recordedPartitionRecords);
+    // Process NewPartitions and track the ones successfully outputted.
+    List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+    List<ByteStringRange> outputtedNewPartitions = new ArrayList<>();
+    for (NewPartition newPartition : newPartitions) {
+      if (processNewPartitionsAction.processNewPartition(newPartition, receiver)) {
+        outputtedNewPartitions.add(newPartition.getPartition());
+      } else {
+        partitionReconciler.addIncompleteNewPartitions(newPartition);
+        orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition);
+      }
     }
 
-    processReconcilerPartitions(receiver, watermarkEstimator, startTime, recordedPartitionRecords);
+    // Process the watermark using read StreamPartitions and NewPartitions.
+    if (streamPartitionsWithWatermark != null) {
+      Optional<Instant> maybeWatermark =
+          getNewWatermark(streamPartitionsWithWatermark, newPartitions);
+      maybeWatermark.ifPresent(metadataTableDao::updateDetectNewPartitionWatermark);
+      // Using NewPartitions and StreamPartitions, evaluate partitions that are possibly not being
+      // streamed. This isn't perfect because there may be partitions moving between
+      // StreamPartitions and NewPartitions while scanning the metadata table. Also, this does not
+      // include NewPartitions marked as deleted from a previous DNP run not yet processed by RCSP.
+      List<ByteStringRange> existingPartitions =
+          streamPartitionsWithWatermark.stream()
+              .map(StreamPartitionWithWatermark::getPartition)
+              .collect(Collectors.toList());
+      existingPartitions.addAll(outputtedNewPartitions);
+      List<ByteStringRange> missingStreamPartitions =
+          getMissingPartitionsFromEntireKeySpace(existingPartitions);
+      orphanedMetadataCleaner.addMissingPartitions(missingStreamPartitions);
+      partitionReconciler.addMissingPartitions(missingStreamPartitions);
+    }
 
-    cleanUpAfterCommit(bundleFinalizer, recordedPartitionRecords);
+    // Only start reconciling after the pipeline has been running for a while.
+    if (tracker.currentRestriction().getFrom() > 50) {
+      processReconcilerPartitions(
+          receiver, watermarkEstimator, initialPipelineState.getStartTime());
+      cleanUpOrphanedMetadata();
+    }
 
-    return ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1));
+    return ProcessContinuation.resume().withResumeDelay(Duration.millis(100));
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsAction.java
index d5a1fc6b74e..15c030f1da3 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsAction.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsAction.java
@@ -18,6 +18,7 @@
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
 
 import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.util.Collections;
 import java.util.List;
 import javax.annotation.Nullable;
 import org.apache.beam.sdk.annotations.Internal;
@@ -25,14 +26,8 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
-import org.apache.beam.sdk.io.range.OffsetRange;
 import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
-import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
-import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.joda.time.Instant;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Class to generate first set of outputs for {@link
@@ -40,8 +35,6 @@ import org.slf4j.LoggerFactory;
  */
 @Internal
 public class GenerateInitialPartitionsAction {
-  private static final Logger LOG = LoggerFactory.getLogger(GenerateInitialPartitionsAction.class);
-
   private final ChangeStreamMetrics metrics;
   private final ChangeStreamDao changeStreamDao;
   @Nullable private final Instant endTime;
@@ -56,35 +49,20 @@ public class GenerateInitialPartitionsAction {
   /**
    * The very first step of the pipeline when there are no partitions being streamed yet. We want to
    * get an initial list of partitions to stream and output them.
-   *
-   * @return {@link ProcessContinuation#resume()} if the stream continues, otherwise {@link
-   *     ProcessContinuation#stop()}
    */
-  public ProcessContinuation run(
-      OutputReceiver<PartitionRecord> receiver,
-      RestrictionTracker<OffsetRange, Long> tracker,
-      ManualWatermarkEstimator<Instant> watermarkEstimator,
-      Instant startTime) {
-    if (!tracker.tryClaim(0L)) {
-      LOG.error(
-          "Could not claim initial DetectNewPartition restriction. No partitions are outputted.");
-      return ProcessContinuation.stop();
-    }
+  public void run(OutputReceiver<PartitionRecord> receiver, Instant startTime) {
     List<ByteStringRange> streamPartitions =
         changeStreamDao.generateInitialChangeStreamPartitions();
-
-    watermarkEstimator.setWatermark(startTime);
-
     for (ByteStringRange partition : streamPartitions) {
       metrics.incListPartitionsCount();
       String uid = UniqueIdGenerator.getNextId();
       PartitionRecord partitionRecord =
-          new PartitionRecord(partition, startTime, uid, startTime, endTime);
+          new PartitionRecord(
+              partition, startTime, uid, startTime, Collections.emptyList(), endTime);
       // We are outputting elements with timestamp of 0 to prevent reliance on event time. This
       // limits the ability to window on commit time of any data changes. It is still possible to
       // window on processing time.
       receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
     }
-    return ProcessContinuation.resume();
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ProcessNewPartitionsAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ProcessNewPartitionsAction.java
new file mode 100644
index 00000000000..e16054a5107
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ProcessNewPartitionsAction.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
+
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.coverSameKeySpace;
+
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.util.Collections;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ProcessNewPartitionsAction {
+  private static final Logger LOG = LoggerFactory.getLogger(ProcessNewPartitionsAction.class);
+
+  private final ChangeStreamMetrics metrics;
+  private final MetadataTableDao metadataTableDao;
+  @Nullable private final Instant endTime;
+
+  public ProcessNewPartitionsAction(
+      ChangeStreamMetrics metrics, MetadataTableDao metadataTableDao, @Nullable Instant endTime) {
+    this.metrics = metrics;
+    this.metadataTableDao = metadataTableDao;
+    this.endTime = endTime;
+  }
+
+  /**
+   * Process a single new partition. New partition resulting from split and merges need to be
+   * outputted to be streamed. Regardless if it's a split or a merge, we have the same verification
+   * process in order to ensure the new partition can actually be streamed.
+   *
+   * <p>When a parent partition splits, it receives two or more new partitions. It will write a new
+   * row, with the new row ranges as row key, for each new partition. These new partitions can be
+   * immediately streamed.
+   *
+   * <p>The complicated scenario is merges. Two or more parent partitions will merge into one new
+   * partition. Each parent partition receives the same new partition (row range) but each parent
+   * partition will have a different continuation token. The parent partitions will all write to the
+   * same row key form by the new row range. Each parent will record its continuation token, and
+   * watermark. Parent partitions may not receive the message to stop at the same time. So when we
+   * try to process the new partition, we need to ensure that all the parent partitions have stopped
+   * and recorded their metadata table. We do so by verifying that the row ranges of the parents
+   * covers a contiguous block of row range that is same as the new row range.
+   *
+   * <p>For example, partition1, A-B, and partition2, B-C, merges into partition3, A-C.
+   *
+   * <ol>
+   *   <li>p1 writes to row A-C in metadata table
+   *   <li>processNewPartition process A-C seeing that only A-B has been recorded and A-B does not
+   *       cover A-C. Do Nothing
+   *   <li>p2 writes to row A-C in metadata table
+   *   <li>processNewPartition process A-C again, seeing that A-B and B-C has been recorded and
+   *       outputs new partition A-C to be streamed.
+   * </ol>
+   *
+   * <p>Note that, the algorithm to verify if a merge is valid, also correctly verifies if a split
+   * is valid. A split is immediately valid as long as the row exists because there's only one
+   * parent that needs to write to that row.
+   *
+   * @param newPartition new partition to be processed
+   * @param receiver to output new partitions
+   */
+  public boolean processNewPartition(
+      NewPartition newPartition, OutputReceiver<PartitionRecord> receiver) {
+    List<ByteStringRange> parentPartitions = newPartition.getParentPartitions();
+    if (!coverSameKeySpace(parentPartitions, newPartition.getPartition())) {
+      LOG.warn("DNP: New partition does not have all the parents: {}", newPartition);
+      return false;
+    }
+
+    String uuid = UniqueIdGenerator.getNextId();
+    PartitionRecord partitionRecord =
+        new PartitionRecord(
+            newPartition.getPartition(),
+            newPartition.getChangeStreamContinuationTokens(),
+            uuid,
+            newPartition.getLowWatermark(),
+            Collections.singletonList(newPartition),
+            endTime);
+    if (parentPartitions.size() > 1) {
+      metrics.incPartitionMergeCount();
+    } else {
+      metrics.incPartitionSplitCount();
+    }
+    LOG.info("DNP: Outputting new partition: {}", partitionRecord);
+    metadataTableDao.markNewPartitionForDeletion(newPartition);
+    // We are outputting elements with timestamp of 0 to prevent reliance on event time. This limits
+    // the ability to window on commit time of any data changes. It is still possible to window on
+    // processing time.
+    receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
+    return true;
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java
index 32262a7d35a..2270cbd08ac 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionAction.java
@@ -17,8 +17,8 @@
  */
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
 
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.coverSameKeySpace;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.isSuperset;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.partitionsToString;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamContinuationTokenHelper.getTokenWithCorrectPartition;
 
@@ -32,6 +32,7 @@ import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
 import com.google.protobuf.ByteString;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import org.apache.beam.sdk.annotations.Internal;
@@ -39,6 +40,7 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.DetectNewPartitionsDoFn;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress;
 import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
@@ -138,48 +140,76 @@ public class ReadChangeStreamPartitionAction {
 
     if (shouldDebug) {
       LOG.info(
-          "RCSP: Partition: "
+          "RCSP {}: Partition: "
               + partitionRecord
               + "\n Watermark: "
               + watermarkEstimator.getState()
               + "\n RestrictionTracker: "
-              + tracker.currentRestriction());
+              + tracker.currentRestriction(),
+          formatByteStringRange(partitionRecord.getPartition()));
     }
 
     // Lock the partition
-    if (!metadataTableDao.lockPartition(
-        partitionRecord.getPartition(), partitionRecord.getUuid())) {
-      LOG.info(
-          "RCSP: Could not acquire lock for partition: {}, with uid: {}, because this is a "
-              + "duplicate and another worker is working on this partition already.",
+    if (tracker.currentRestriction().isEmpty()) {
+      boolean lockedPartition = metadataTableDao.lockAndRecordPartition(partitionRecord);
+      // Clean up NewPartition on the first run regardless of locking result. If locking fails it
+      // means this partition is being streamed, then cleaning up NewPartitions avoids lingering
+      // NewPartitions.
+      for (NewPartition newPartition : partitionRecord.getParentPartitions()) {
+        metadataTableDao.deleteNewPartition(newPartition);
+      }
+      if (!lockedPartition) {
+        LOG.info(
+            "RCSP  {} : Could not acquire lock with uid: {}, because this is a "
+                + "duplicate and another worker is working  on this partition already.",
+            formatByteStringRange(partitionRecord.getPartition()),
+            partitionRecord.getUuid());
+        StreamProgress streamProgress = new StreamProgress();
+        streamProgress.setFailToLock(true);
+        metrics.decPartitionStreamCount();
+        tracker.tryClaim(streamProgress);
+        return ProcessContinuation.stop();
+      }
+    } else if (tracker.currentRestriction().getCloseStream() == null
+        && !metadataTableDao.doHoldLock(
+            partitionRecord.getPartition(), partitionRecord.getUuid())) {
+      // We only verify the lock if we are not holding CloseStream because if this is a retry of
+      // CloseStream we might have already cleaned up the lock in a previous attempt.
+      // Failed correctness check on this worker holds the lock on this partition. This shouldn't
+      // fail because there's a restriction tracker which means this worker has already acquired the
+      // lock and once it has acquired the lock it shouldn't fail the lock check.
+      LOG.warn(
+          "RCSP  {} : Subsequent run that doesn't hold the lock {}. This is not unexpected and "
+              + "should probably be reviewed.",
           formatByteStringRange(partitionRecord.getPartition()),
           partitionRecord.getUuid());
       StreamProgress streamProgress = new StreamProgress();
       streamProgress.setFailToLock(true);
       metrics.decPartitionStreamCount();
-      if (!tracker.tryClaim(streamProgress)) {
-        LOG.debug("RCSP: Failed to claim tracker after failing to lock partition.");
-        return ProcessContinuation.stop();
-      }
+      tracker.tryClaim(streamProgress);
       return ProcessContinuation.stop();
     }
 
     // Process CloseStream if it exists
     CloseStream closeStream = tracker.currentRestriction().getCloseStream();
     if (closeStream != null) {
+      LOG.debug("RCSP: Processing CloseStream");
+      metrics.decPartitionStreamCount();
       if (closeStream.getStatus().getCode() == Status.Code.OK) {
         // We need to update watermark here. We're terminating this stream because we have reached
         // endTime. Instant.now is greater or equal to endTime. The goal here is
         // DNP will need to know this stream has passed the endTime so DNP can eventually terminate.
-        watermarkEstimator.setWatermark(Instant.now());
+        Instant terminatingWatermark = Instant.ofEpochMilli(Long.MAX_VALUE);
+        Instant endTime = partitionRecord.getEndTime();
+        if (endTime != null) {
+          terminatingWatermark = endTime;
+        }
+        watermarkEstimator.setWatermark(terminatingWatermark);
         metadataTableDao.updateWatermark(
-            partitionRecord.getPartition(),
-            watermarkEstimator.currentWatermark(),
-            tracker.currentRestriction().getCurrentToken());
+            partitionRecord.getPartition(), watermarkEstimator.currentWatermark(), null);
         LOG.info(
             "RCSP {}: Reached end time, terminating...",
             formatByteStringRange(partitionRecord.getPartition()));
-        metrics.decPartitionStreamCount();
         return ProcessContinuation.stop();
       }
       if (closeStream.getStatus().getCode() != Status.Code.OUT_OF_RANGE) {
@@ -187,14 +217,27 @@ public class ReadChangeStreamPartitionAction {
             "RCSP {}: Reached unexpected terminal state: {}",
             formatByteStringRange(partitionRecord.getPartition()),
             closeStream.getStatus());
-        metrics.decPartitionStreamCount();
         return ProcessContinuation.stop();
       }
-      // The partitions in the continuation tokens should be a superset of this partition.
-      // If there's only 1 token, then the token's partition should be a superset of this partition.
-      // If there are more than 1 tokens, then the tokens should form a continuous row range that is
-      // a superset of this partition.
+      // Release the lock only if the uuid matches. In normal operation this doesn't change
+      // anything. However, it's possible for this RCSP to crash while processing CloseStream but
+      // after the side effects of writing the new partitions to the metadata table. New partitions
+      // can be created while this RCSP restarts from the previous checkpoint and processes the
+      // CloseStream again. In certain race scenarios the child partitions may merge back to this
+      // partition, but as a new RCSP. The new partition (same as this partition) would write the
+      // exact same content to the metadata table but with a different uuid. We don't want to
+      // accidentally delete the StreamPartition because it now belongs to the new RCSP.
+      // If the uuid is the same (meaning this race scenario did not take place) we release the lock
+      // and mark the StreamPartition to be deleted, so we can delete it after we have written the
+      // NewPartitions.
+      metadataTableDao.releaseStreamPartitionLockForDeletion(
+          partitionRecord.getPartition(), partitionRecord.getUuid());
+      // The partitions in the continuation tokens must cover the same key space as this partition.
+      // If there's only 1 token, then the token's partition is equals to this partition.
+      // If there are more than 1 tokens, then the tokens form a continuous row range equals to this
+      // partition.
       List<ByteStringRange> childPartitions = new ArrayList<>();
+      List<ByteStringRange> tokenPartitions = new ArrayList<>();
       // Check if NewPartitions field exists, if not we default to using just the
       // ChangeStreamContinuationTokens.
       boolean useNewPartitionsField =
@@ -212,8 +255,10 @@ public class ReadChangeStreamPartitionAction {
             getTokenWithCorrectPartition(
                 partitionRecord.getPartition(),
                 closeStream.getChangeStreamContinuationTokens().get(i));
+        tokenPartitions.add(token.getPartition());
         metadataTableDao.writeNewPartition(
-            childPartition, token, partitionRecord.getPartition(), watermarkEstimator.getState());
+            new NewPartition(
+                childPartition, Collections.singletonList(token), watermarkEstimator.getState()));
       }
       if (shouldDebug) {
         LOG.info(
@@ -221,14 +266,16 @@ public class ReadChangeStreamPartitionAction {
             formatByteStringRange(partitionRecord.getPartition()),
             partitionsToString(childPartitions));
       }
-      if (!isSuperset(childPartitions, partitionRecord.getPartition())) {
+      if (!coverSameKeySpace(tokenPartitions, partitionRecord.getPartition())) {
         LOG.warn(
-            "RCSP {}: CloseStream has child partition(s) {} that doesn't cover the keyspace",
+            "RCSP {}: CloseStream has tokens {} that don't cover the entire keyspace",
             formatByteStringRange(partitionRecord.getPartition()),
-            partitionsToString(childPartitions));
+            partitionsToString(tokenPartitions));
       }
+      // Perform the real cleanup. This step is no op if the race mentioned above occurs (splits and
+      // merges results back to this partition again) because when we register the "new" partition,
+      // we unset the deletion bit.
       metadataTableDao.deleteStreamPartitionRow(partitionRecord.getPartition());
-      metrics.decPartitionStreamCount();
       return ProcessContinuation.stop();
     }
 
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineAction.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineAction.java
new file mode 100644
index 00000000000..79d6ae90a25
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineAction.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.HashMap;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
+import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ResumeFromPreviousPipelineAction {
+  private static final Logger LOG = LoggerFactory.getLogger(ResumeFromPreviousPipelineAction.class);
+
+  private final ChangeStreamMetrics metrics;
+  private final MetadataTableDao metadataTableDao;
+  @Nullable private final Instant endTime;
+  private final ProcessNewPartitionsAction processNewPartitionsAction;
+
+  public ResumeFromPreviousPipelineAction(
+      ChangeStreamMetrics metrics,
+      MetadataTableDao metadataTableDao,
+      @Nullable Instant endTime,
+      ProcessNewPartitionsAction processNewPartitionsAction) {
+    this.metrics = metrics;
+    this.metadataTableDao = metadataTableDao;
+    this.endTime = endTime;
+    this.processNewPartitionsAction = processNewPartitionsAction;
+  }
+
+  /**
+   * Resume from previously drained pipeline. Restart StreamPartitions by reading the last metadata
+   * write and output PartitionRecord based on that. Process NewPartitions normally.
+   *
+   * <ol>
+   *   <li>Clear out missing partitions metadata in the metadata table
+   *   <li>Read all StreamPartitions and output them
+   *   <li>Read all NewPartitions and output them normally
+   * </ol>
+   *
+   * @param receiver output receiver to output
+   * @throws InvalidProtocolBufferException when failing to read from metadata table.
+   */
+  public void run(OutputReceiver<PartitionRecord> receiver) throws InvalidProtocolBufferException {
+    // Erase any existing missing partitions.
+    metadataTableDao.writeDetectNewPartitionMissingPartitions(new HashMap<>());
+
+    List<PartitionRecord> partitions = metadataTableDao.readAllStreamPartitions();
+
+    for (PartitionRecord partitionRecord : partitions) {
+      if (partitionRecord.getUuid().isEmpty()) {
+        partitionRecord.setUuid(UniqueIdGenerator.getNextId());
+      }
+      if (endTime != null) {
+        partitionRecord.setEndTime(endTime);
+      }
+      LOG.info("DNP: Outputting existing partition: {}", partitionRecord);
+      metrics.incListPartitionsCount();
+      receiver.outputWithTimestamp(partitionRecord, Instant.EPOCH);
+    }
+
+    List<NewPartition> newPartitions = metadataTableDao.readNewPartitionsIncludingDeleted();
+
+    for (NewPartition newPartition : newPartitions) {
+      processNewPartitionsAction.processNewPartition(newPartition, receiver);
+    }
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java
index 29d8d72d9f6..69c6efba1de 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/BigtableChangeStreamAccessor.java
@@ -110,12 +110,6 @@ class BigtableChangeStreamAccessor {
             EnhancedBigtableStubSettings.defaultGrpcTransportProviderBuilder()
                 .setAttemptDirectPath(false) // Disable DirectPath
                 .setChannelPoolSettings( // Autoscale Channel Size
-                    // We use blocking rpc calls so we expect a max of `workerHarnessThreads`
-                    // concurrent rpcs per-worker. This defaults to 500 and we want to accommodate
-                    // higher concurrency for jobs that are overridden to use more threads. There
-                    // is a limit of 100 concurrent streams per channel so this should resize to
-                    // support a large number of requests. Testing has shown that this
-                    // significantly increases throughput compared to the default channel pool.
                     ChannelPoolSettings.builder()
                         // Make sure that there are at least 2 channels regardless of RPCs
                         .setMinChannelCount(2)
@@ -191,9 +185,6 @@ class BigtableChangeStreamAccessor {
         .readChangeStreamSettings()
         .setRetrySettings(
             readChangeStreamRetrySettings
-                // Set timeouts to 60s - dataflow should checkpoint before then, but it is not
-                // guaranteed to happen after a specific duration. We still want a conservative
-                // timeout so that it can't hang.
                 .setInitialRpcTimeout(Duration.ofSeconds(60))
                 .setTotalTimeout(Duration.ofSeconds(60))
                 .setMaxRpcTimeout(Duration.ofSeconds(60))
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java
index f47d0f6370e..1c24e09db7d 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/ChangeStreamDao.java
@@ -89,7 +89,6 @@ public class ChangeStreamDao {
       query.continuationTokens(Collections.singletonList(currentToken));
     } else if (startTime != null) {
       query.startTime(toThreetenInstant(startTime));
-      // Check if partition has Continuation Tokens
     } else if (changeStreamContinuationTokenList != null) {
       query.continuationTokens(changeStreamContinuationTokenList);
     } else {
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java
index b826789fef4..acecc22b891 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/DaoFactory.java
@@ -92,11 +92,10 @@ public class DaoFactory implements Serializable {
     if (changeStreamDao == null) {
       checkArgumentNotNull(changeStreamConfig.getProjectId());
       checkArgumentNotNull(changeStreamConfig.getInstanceId());
-      String tableId = this.tableId;
       checkArgumentNotNull(changeStreamConfig.getAppProfileId());
       BigtableDataClient dataClient =
           BigtableChangeStreamAccessor.getOrCreate(changeStreamConfig).getDataClient();
-      changeStreamDao = new ChangeStreamDao(dataClient, tableId);
+      changeStreamDao = new ChangeStreamDao(dataClient, this.tableId);
     }
     return changeStreamDao;
   }
@@ -122,7 +121,7 @@ public class DaoFactory implements Serializable {
     if (metadataTableAdminDao == null) {
       checkArgumentNotNull(metadataTableConfig.getProjectId());
       checkArgumentNotNull(metadataTableConfig.getInstanceId());
-      String tableId = checkArgumentNotNull(this.metadataTableId);
+      checkArgumentNotNull(this.metadataTableId);
       checkArgumentNotNull(metadataTableConfig.getAppProfileId());
       BigtableTableAdminClient tableAdminClient =
           BigtableChangeStreamAccessor.getOrCreate(metadataTableConfig).getTableAdminClient();
@@ -130,7 +129,7 @@ public class DaoFactory implements Serializable {
           BigtableChangeStreamAccessor.getOrCreate(metadataTableConfig).getInstanceAdminClient();
       metadataTableAdminDao =
           new MetadataTableAdminDao(
-              tableAdminClient, instanceAdminClient, changeStreamName, tableId);
+              tableAdminClient, instanceAdminClient, changeStreamName, this.metadataTableId);
     }
     return metadataTableAdminDao;
   }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java
index d2f3872d329..5194ca49d62 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableAdminDao.java
@@ -34,10 +34,10 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Immutabl
 /**
  * Data access object for creating and dropping the metadata table.
  *
- * <p>The metadata table will be used to keep the state of the entire Dataflow job as well as
+ * <p>The metadata table will be used to keep the state of the entire Beam pipeline as well as
  * splitting and merging partitions.
  *
- * <p>Each Dataflow job will create its own metadata table.
+ * <p>Each Beam pipeline will create its own metadata table.
  */
 @Internal
 public class MetadataTableAdminDao {
@@ -50,6 +50,7 @@ public class MetadataTableAdminDao {
   public static final String CF_LOCK = "lock";
   public static final String CF_MISSING_PARTITIONS = "missing_partitions";
   public static final String CF_VERSION = "version";
+  public static final String CF_SHOULD_DELETE = "should_delete";
   public static final String QUALIFIER_DEFAULT = "latest";
   public static final ImmutableList<String> COLUMN_FAMILIES =
       ImmutableList.of(
@@ -60,7 +61,8 @@ public class MetadataTableAdminDao {
           CF_CONTINUATION_TOKEN,
           CF_LOCK,
           CF_MISSING_PARTITIONS,
-          CF_VERSION);
+          CF_VERSION,
+          CF_SHOULD_DELETE);
   public static final ByteString NEW_PARTITION_PREFIX = ByteString.copyFromUtf8("NewPartition#");
   public static final ByteString STREAM_PARTITION_PREFIX =
       ByteString.copyFromUtf8("StreamPartition#");
@@ -170,4 +172,32 @@ public class MetadataTableAdminDao {
     tableAdminClient.createTable(createTableRequest);
     return true;
   }
+
+  /**
+   * Delete all the metadata rows starting with the change stream name prefix, except for detect new
+   * partition row because it signals the existence of a pipeline with the change stream name.
+   */
+  public void cleanUpPrefix() {
+    // Clean up all rows except for DNP row.
+    tableAdminClient.dropRowRange(tableId, getFullNewPartitionPrefix());
+    tableAdminClient.dropRowRange(tableId, getFullStreamPartitionPrefix());
+  }
+
+  /**
+   * Return new partition row key prefix concatenated with change stream name.
+   *
+   * @return new partition row key prefix concatenated with change stream name.
+   */
+  private ByteString getFullNewPartitionPrefix() {
+    return changeStreamNamePrefix.concat(NEW_PARTITION_PREFIX);
+  }
+
+  /**
+   * Return stream partition row key prefix concatenated with change stream name.
+   *
+   * @return stream partition row key prefix concatenated with change stream name.
+   */
+  private ByteString getFullStreamPartitionPrefix() {
+    return changeStreamNamePrefix.concat(STREAM_PARTITION_PREFIX);
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java
index 30abd6d88a1..295184e1cc1 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDao.java
@@ -18,28 +18,50 @@
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao;
 
 import static com.google.cloud.bigtable.data.v2.models.Filters.FILTERS;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.microsecondToInstant;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao.NEW_PARTITION_PREFIX;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao.STREAM_PARTITION_PREFIX;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.isRowLocked;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseInitialContinuationTokens;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseLockUuid;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseTokenFromRow;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseWatermarkFromRow;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseWatermarkLastUpdatedFromRow;
 
 import com.google.api.gax.rpc.ServerStream;
 import com.google.cloud.bigtable.data.v2.BigtableDataClient;
 import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
 import com.google.cloud.bigtable.data.v2.models.ConditionalRowMutation;
+import com.google.cloud.bigtable.data.v2.models.Filters;
 import com.google.cloud.bigtable.data.v2.models.Filters.Filter;
 import com.google.cloud.bigtable.data.v2.models.Mutation;
 import com.google.cloud.bigtable.data.v2.models.Query;
 import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
 import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowCell;
 import com.google.cloud.bigtable.data.v2.models.RowMutation;
 import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.apache.beam.repackaged.core.org.apache.commons.lang3.SerializationException;
 import org.apache.beam.repackaged.core.org.apache.commons.lang3.SerializationUtils;
 import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.DetectNewPartitionsState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.StreamPartitionWithWatermark;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs;
+import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -54,6 +76,11 @@ import org.slf4j.LoggerFactory;
 public class MetadataTableDao {
   private static final Logger LOG = LoggerFactory.getLogger(MetadataTableDao.class);
 
+  // New partition marked for deletion should be reevaluated after this delay because it might be
+  // output has failed.
+  private static final Duration DELETED_NEW_PARTITION_REEVALUATE_DELAY =
+      Duration.standardMinutes(1);
+
   private final BigtableDataClient dataClient;
   private final String tableId;
   private final ByteString changeStreamNamePrefix;
@@ -150,40 +177,123 @@ public class MetadataTableDao {
   }
 
   /**
-   * @return stream of all the new partitions resulting from splits and merges waiting to be
-   *     streamed.
+   * Read the low watermark of the pipeline from Detect New Partition row.
+   *
+   * @return DetectNewPartitions row from the metadata table.
+   */
+  public @Nullable DetectNewPartitionsState readDetectNewPartitionsState() {
+    Row row =
+        dataClient.readRow(
+            tableId,
+            getFullDetectNewPartition(),
+            FILTERS
+                .chain()
+                .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_WATERMARK))
+                .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
+                .filter(FILTERS.limit().cellsPerColumn(1)));
+    if (row == null) {
+      return null;
+    }
+    Instant watermark = parseWatermarkFromRow(row);
+    Instant timestamp = parseWatermarkLastUpdatedFromRow(row);
+    if (watermark == null || timestamp == null) {
+      return null;
+    }
+    return new DetectNewPartitionsState(watermark, timestamp);
+  }
+
+  /**
+   * @return all the new partitions resulting from splits and merges waiting to be streamed
+   *     including ones marked for deletion.
    */
-  public ServerStream<Row> readNewPartitions() {
-    // It's important that we limit to the latest value per column because it's possible to write to
-    // the same column multiple times. We don't want to read and send duplicate tokens to the
-    // server.
+  public List<NewPartition> readNewPartitionsIncludingDeleted()
+      throws InvalidProtocolBufferException {
+    List<NewPartition> newPartitions = new ArrayList<>();
     Query query =
         Query.create(tableId)
             .prefix(getFullNewPartitionPrefix())
-            .filter(FILTERS.limit().cellsPerColumn(1));
-    return dataClient.readRows(query);
+            .filter(Filters.FILTERS.limit().cellsPerColumn(1));
+    ServerStream<Row> rows = dataClient.readRows(query);
+    for (Row row : rows) {
+      ByteStringRange childPartition = convertNewPartitionRowKeyToPartition(row.getKey());
+      List<ChangeStreamContinuationToken> tokens = new ArrayList<>();
+      // Capture all the initial continuation tokens to request for the new partition.
+      for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_INITIAL_TOKEN)) {
+        ChangeStreamContinuationToken changeStreamContinuationToken =
+            ChangeStreamContinuationToken.fromByteString(cell.getValue());
+        tokens.add(changeStreamContinuationToken);
+      }
+      // There are no parents.
+      if (tokens.isEmpty()) {
+        continue;
+      }
+      // Capture all the low watermark and calculate the low watermark among all the parents.
+      List<Long> parentLowWatermark = new ArrayList<>();
+      for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS)) {
+        parentLowWatermark.add(Longs.fromByteArray(cell.getValue().toByteArray()));
+      }
+      Instant lowWatermark = Instant.ofEpochMilli(Collections.min(parentLowWatermark));
+      newPartitions.add(new NewPartition(childPartition, tokens, lowWatermark));
+    }
+    return newPartitions;
   }
 
-  /**
-   * After a split or merge from a close stream, write the new partition's information to the
-   * metadata table.
-   *
-   * @param newPartition the new partition
-   * @param changeStreamContinuationToken the token that can be used to pick up from where the
-   *     parent left off
-   * @param parentPartition the parent that stopped and split or merged
-   * @param lowWatermark the low watermark of the parent stream
-   */
-  public void writeNewPartition(
-      ByteStringRange newPartition,
-      ChangeStreamContinuationToken changeStreamContinuationToken,
-      ByteStringRange parentPartition,
-      Instant lowWatermark) {
-    writeNewPartition(
-        newPartition,
-        changeStreamContinuationToken.toByteString(),
-        ByteStringRange.serializeToByteString(parentPartition),
-        lowWatermark);
+  /** @return list the new partitions resulting from splits and merges waiting to be streamed. */
+  public List<NewPartition> readNewPartitions() throws InvalidProtocolBufferException {
+    List<NewPartition> newPartitions = new ArrayList<>();
+    Query query =
+        Query.create(tableId)
+            .prefix(getFullNewPartitionPrefix())
+            .filter(Filters.FILTERS.limit().cellsPerColumn(1));
+    ServerStream<Row> rows = dataClient.readRows(query);
+    for (Row row : rows) {
+      long lastUpdatedMicro = 0;
+      // We only process parent partition that are not marked for deletion.
+      Set<ByteStringRange> deletedParentPartitions = new HashSet<>();
+      for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_SHOULD_DELETE)) {
+        lastUpdatedMicro = Math.max(lastUpdatedMicro, cell.getTimestamp());
+        // Evaluate if the timestamp of the deletion marker is within the last 1 minute.
+        if (microsecondToInstant(cell.getTimestamp())
+            .plus(DELETED_NEW_PARTITION_REEVALUATE_DELAY)
+            .isAfterNow()) {
+          deletedParentPartitions.add(ByteStringRange.toByteStringRange(cell.getQualifier()));
+        }
+      }
+
+      ByteStringRange childPartition = convertNewPartitionRowKeyToPartition(row.getKey());
+      List<ChangeStreamContinuationToken> tokens = new ArrayList<>();
+      // Capture all the initial continuation tokens to request for the new partition.
+      for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_INITIAL_TOKEN)) {
+        // Only add parent partition that are not deleted.
+        if (!deletedParentPartitions.contains(
+            ByteStringRange.toByteStringRange(cell.getQualifier()))) {
+          ChangeStreamContinuationToken changeStreamContinuationToken =
+              ChangeStreamContinuationToken.fromByteString(cell.getValue());
+          tokens.add(changeStreamContinuationToken);
+          lastUpdatedMicro = Math.max(lastUpdatedMicro, cell.getTimestamp());
+        }
+      }
+      // All the parent partitions were skipped. We can skip this row.
+      if (tokens.isEmpty()) {
+        continue;
+      }
+
+      // Capture all the low watermark and calculate the low watermark among all the parents.
+      List<Long> parentLowWatermark = new ArrayList<>();
+      for (RowCell cell : row.getCells(MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS)) {
+        // Only add parent partition that are not deleted.
+        if (!deletedParentPartitions.contains(
+            ByteStringRange.toByteStringRange(cell.getQualifier()))) {
+          parentLowWatermark.add(Longs.fromByteArray(cell.getValue().toByteArray()));
+          lastUpdatedMicro = Math.max(lastUpdatedMicro, cell.getTimestamp());
+        }
+      }
+      Instant lowWatermark = Instant.ofEpochMilli(Collections.min(parentLowWatermark));
+      newPartitions.add(
+          new NewPartition(
+              childPartition, tokens, lowWatermark, microsecondToInstant(lastUpdatedMicro)));
+    }
+    return newPartitions;
   }
 
   /**
@@ -191,45 +301,193 @@ public class MetadataTableDao {
    * metadata table.
    *
    * @param newPartition the new partition
-   * @param newPartitionContinuationToken continuation token for the new partition
-   * @param parentPartition the parent that stopped
-   * @param lowWatermark low watermark of the parent
    */
-  private void writeNewPartition(
-      ByteStringRange newPartition,
-      ByteString newPartitionContinuationToken,
-      ByteString parentPartition,
-      Instant lowWatermark) {
-    LOG.debug("Insert new partition");
-    ByteString rowKey = convertPartitionToNewPartitionRowKey(newPartition);
+  public void writeNewPartition(NewPartition newPartition) {
+    ByteString rowKey = convertPartitionToNewPartitionRowKey(newPartition.getPartition());
+    ByteStringRange parentPartition =
+        newPartition.getChangeStreamContinuationTokens().get(0).getPartition();
     RowMutation rowMutation =
         RowMutation.create(tableId, rowKey)
-            .setCell(MetadataTableAdminDao.CF_INITIAL_TOKEN, newPartitionContinuationToken, 1)
-            .setCell(MetadataTableAdminDao.CF_PARENT_PARTITIONS, parentPartition, 1)
+            .setCell(
+                MetadataTableAdminDao.CF_INITIAL_TOKEN,
+                ByteStringRange.serializeToByteString(parentPartition),
+                newPartition.getChangeStreamContinuationTokens().get(0).toByteString())
             .setCell(
                 MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS,
-                parentPartition,
-                lowWatermark.getMillis());
+                ByteStringRange.serializeToByteString(parentPartition),
+                newPartition.getLowWatermark().getMillis())
+            .deleteCells(
+                MetadataTableAdminDao.CF_SHOULD_DELETE,
+                ByteStringRange.serializeToByteString(parentPartition));
     dataClient.mutateRow(rowMutation);
   }
 
   /**
-   * @return stream of partitions currently being streamed by the beam job that have set a
-   *     watermark.
+   * This is the 1st step of 2 phase delete. Mark each parent partition in NewPartition for
+   * deletion. This avoids double reading the new partitions row for a while to allow RCSP to clean
+   * up partitions that led to it.
+   *
+   * <p>The reason behind 2 phase delete of NewPartition is to provide the invariant that the
+   * metadata table always has a copy of the ChangeStreamContinuationToken for every partition. The
+   * alternatives are either delete NewPartition <i>before</i> outputting PartitionRecord to RCSP or
+   * delete NewPartition <i>after</i> outputting PartitionRecord to RCSP.
+   *
+   * <p>The former has the problem that if the output failed and the NewPartition row has been
+   * deleted, the token is now lost.
+   *
+   * <p>The latter has a more complex problem of race between cleaning up NewPartition and RCSP
+   * writing StreamPartition. If clean up happens before StreamPartition is written, then at that
+   * moment the token is missing. While under normal operations, it should recover because RCSP
+   * writes StreamPartition eventually. We want to provide the invariant that there's always a copy
+   * of a token for every partition in the metadata table.
+   *
+   * @param newPartition mark for deletion.
    */
-  public ServerStream<Row> readFromMdTableStreamPartitionsWithWatermark() {
-    // We limit to the latest value per column.
+  public void markNewPartitionForDeletion(NewPartition newPartition) {
+    ByteString rowKey = convertPartitionToNewPartitionRowKey(newPartition.getPartition());
+    RowMutation rowMutation = RowMutation.create(tableId, rowKey);
+    for (ChangeStreamContinuationToken token : newPartition.getChangeStreamContinuationTokens()) {
+      rowMutation.setCell(
+          MetadataTableAdminDao.CF_SHOULD_DELETE,
+          ByteStringRange.serializeToByteString(token.getPartition()),
+          1);
+    }
+    dataClient.mutateRow(rowMutation);
+  }
+
+  /**
+   * This is the 2nd step of 2 phase delete. Delete the newPartition cells. This should take place
+   * after the tokens have been written to StreamPartition.
+   *
+   * <p>It's possible to try to delete multiple parent partition cells but only a subset are marked
+   * for deletion. Only the cells marked for deletion are deleted.
+   *
+   * @param newPartition row that represents the new partition.
+   * @return true if successfully deleted entire newPartition.
+   */
+  public boolean deleteNewPartition(NewPartition newPartition) {
+    ByteString rowKey = convertPartitionToNewPartitionRowKey(newPartition.getPartition());
+
+    boolean success = true;
+
+    for (ChangeStreamContinuationToken token : newPartition.getChangeStreamContinuationTokens()) {
+      Filter shouldDelete =
+          FILTERS
+              .chain()
+              .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_SHOULD_DELETE))
+              .filter(
+                  FILTERS
+                      .qualifier()
+                      .exactMatch(ByteStringRange.serializeToByteString(token.getPartition())))
+              .filter(FILTERS.value().exactMatch(ByteString.copyFrom(Longs.toByteArray(1))));
+      Mutation mutation =
+          Mutation.create()
+              .deleteCells(
+                  MetadataTableAdminDao.CF_INITIAL_TOKEN,
+                  ByteStringRange.serializeToByteString(token.getPartition()))
+              .deleteCells(
+                  MetadataTableAdminDao.CF_PARENT_LOW_WATERMARKS,
+                  ByteStringRange.serializeToByteString(token.getPartition()))
+              .deleteCells(
+                  MetadataTableAdminDao.CF_SHOULD_DELETE,
+                  ByteStringRange.serializeToByteString(token.getPartition()));
+
+      ConditionalRowMutation conditionalRowMutation =
+          ConditionalRowMutation.create(tableId, rowKey).condition(shouldDelete).then(mutation);
+      success = dataClient.checkAndMutateRow(conditionalRowMutation) && success;
+    }
+
+    return success;
+  }
+
+  /**
+   * Return list of locked StreamPartition and their watermarks.
+   *
+   * @return list of partitions currently being streamed that have a watermark.
+   */
+  public List<StreamPartitionWithWatermark> readStreamPartitionsWithWatermark()
+      throws InvalidProtocolBufferException {
+    LOG.debug(
+        "Reading stream partitions from metadata table: "
+            + getFullStreamPartitionPrefix().toStringUtf8());
+    Filter filterForWatermark =
+        FILTERS
+            .chain()
+            .filter(Filters.FILTERS.limit().cellsPerColumn(1))
+            .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_WATERMARK))
+            .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT));
+    Filter filterForLock =
+        FILTERS
+            .chain()
+            .filter(Filters.FILTERS.limit().cellsPerColumn(1))
+            .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_LOCK))
+            .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT));
     Query query =
         Query.create(tableId)
             .prefix(getFullStreamPartitionPrefix())
-            .filter(
-                FILTERS
-                    .chain()
-                    .filter(FILTERS.limit().cellsPerColumn(1))
-                    .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_WATERMARK))
-                    .filter(
-                        FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT)));
-    return dataClient.readRows(query);
+            .filter(FILTERS.interleave().filter(filterForWatermark).filter(filterForLock));
+    ServerStream<Row> rows = dataClient.readRows(query);
+    List<StreamPartitionWithWatermark> partitions = new ArrayList<>();
+    for (Row row : rows) {
+      if (!isRowLocked(row)) {
+        continue;
+      }
+      Instant watermark = MetadataTableEncoder.parseWatermarkFromRow(row);
+      if (watermark == null) {
+        continue;
+      }
+      ByteStringRange partition = convertStreamPartitionRowKeyToPartition(row.getKey());
+      partitions.add(new StreamPartitionWithWatermark(partition, watermark));
+    }
+    return partitions;
+  }
+
+  /**
+   * Read all the StreamPartition and output PartitionRecord to stream them.
+   *
+   * @return list of PartitionRecord of all StreamPartitions in the metadata table.
+   */
+  public List<PartitionRecord> readAllStreamPartitions() throws InvalidProtocolBufferException {
+    Query query = Query.create(tableId).prefix(getFullStreamPartitionPrefix());
+    ServerStream<Row> rows = dataClient.readRows(query);
+    List<PartitionRecord> partitions = new ArrayList<>();
+    for (Row row : rows) {
+      Instant watermark = parseWatermarkFromRow(row);
+      String uuid = parseLockUuid(row);
+      ByteStringRange partition = convertStreamPartitionRowKeyToPartition(row.getKey());
+      String currentToken = parseTokenFromRow(row);
+      List<ChangeStreamContinuationToken> initialTokens = parseInitialContinuationTokens(row);
+      if (watermark == null) {
+        // This is unexpected. Watermark should never be null. We can't just leave this around
+        // either, it will cause conflicts. We need to delete it.
+        if (uuid != null) {
+          releaseStreamPartitionLockForDeletion(partition, uuid);
+        }
+        deleteStreamPartitionRow(partition);
+        LOG.error("Cleaning up corrupted StreamPartition {}", formatByteStringRange(partition));
+        continue;
+      }
+      PartitionRecord partitionRecord;
+      if (currentToken != null) {
+        ChangeStreamContinuationToken token =
+            ChangeStreamContinuationToken.create(partition, currentToken);
+        partitionRecord =
+            new PartitionRecord(
+                partition, Collections.singletonList(token), watermark, Collections.emptyList());
+      } else if (!initialTokens.isEmpty()) {
+        partitionRecord =
+            new PartitionRecord(partition, initialTokens, watermark, Collections.emptyList());
+      } else {
+        partitionRecord =
+            new PartitionRecord(partition, watermark, watermark, Collections.emptyList());
+      }
+
+      if (uuid != null) {
+        partitionRecord.setUuid(uuid);
+      }
+      partitions.add(partitionRecord);
+    }
+    return partitions;
   }
 
   /**
@@ -256,6 +514,15 @@ public class MetadataTableDao {
     dataClient.mutateRow(rowMutation);
   }
 
+  /**
+   * Update the watermark cell for Detect New Partition step.
+   *
+   * @param watermark watermark value to set for the cell
+   */
+  public void updateDetectNewPartitionWatermark(Instant watermark) {
+    writeToMdTableWatermarkHelper(getFullDetectNewPartition(), watermark, null);
+  }
+
   /**
    * Update the metadata for the row key represented by the partition.
    *
@@ -272,41 +539,103 @@ public class MetadataTableDao {
   }
 
   /**
-   * Delete the row key represented by the partition. This represents that the partition will no
-   * longer be streamed.
+   * This is the 1st step of 2 phase delete of StreamPartition. Unset the uuid of StreamPartition if
+   * the uuid matches and set the deletion bit. This prepares the StreamPartition to be deleted.
    *
-   * @param partition forms the row key of the row to delete
+   * <p>The reason to have 2 phase delete is
+   *
+   * <ul>
+   *   <li>If we delete StreamPartition <b>before</b> we write NewPartitions, then we can't satisfy
+   *       the invariant that there always exists a continuation token for every row key; no matter
+   *       how temporary.
+   *   <li>If we delete StreamPartition <b>after</b> we write NewPartitions, we allow a more complex
+   *       and rare race problem. After we write the NewPartition but before we delete
+   *       StreamPartition the following series of event take place. DNP processes the
+   *       NewPartitions. The NewPartitions start streaming and then immediately merge back to the
+   *       same partition as the partition we're about to Delete. The new partition tries to lock
+   *       the StreamPartition row. Locking fails because the lock is held by the RCSP that we're
+   *       deleting.
+   *       <p>For example: the RCSP with id <code>1234</code> is working on partition AC. Metadata
+   *       table has the row <code>
+   *       StreamPartition#AC</code> with <code>uuid=1234</code>. Partition AC splits into AB and
+   *       BC. We write <code>NewPartition#AB</code> and <code>
+   *       NewPartition#BC</code>. But we haven't deleted <code>StreamPartition#AC</code> yet.
+   *       Before we clean up <code>StreamPartition#AC</code>, DNP processes AB and BC. AB and BC
+   *       start streaming. AB and BC immediately merge back to AC. DNP processes the <code>
+   *       NewPartition#AC</code> and outputs AC. A new RCSP with id 5678 tries to lock <code>
+   *       StreamPartition#AC</code> to start processing AC. This fails because RCSP 1234 still
+   *       holds the lock. Once the lock fails RCSP 5678 stops. RCSP 1234 eventually cleans up
+   *       <code>StreamPartition#AC</code> and no RCSP is streaming AC anymore.
+   * </ul>
+   *
+   * The solution is a 2 phase delete
+   *
+   * <ol>
+   *   <li>Release the lock and mark the StreamPartition for deletion
+   *   <li>Write NewPartition rows
+   *   <li>Delete StreamPartition if and only if deletion bit is set
+   * </ol>
+   *
+   * We added the deletion bit to prevent accidentally deleting a new StreamPartition as the result
+   * of the split and merge race described above. If the above scenario happens, the new partition
+   * will be able to lock the StreamPartition row. But we don't want to delete the StreamPartition,
+   * so we check if it is still marked for deletion. Obviously, locking the row unsets the deletion
+   * bit, so it can't be deleted.
+   *
+   * @param partition release the lock for this partition
+   * @param uuid match the uuid
+   * @return true if releasing the lock was successful.
    */
-  public void deleteStreamPartitionRow(ByteStringRange partition) {
+  public boolean releaseStreamPartitionLockForDeletion(ByteStringRange partition, String uuid) {
     ByteString rowKey = convertPartitionToStreamPartitionRowKey(partition);
-    RowMutation rowMutation = RowMutation.create(tableId, rowKey).deleteRow();
-    dataClient.mutateRow(rowMutation);
+    Filter lockCellFilter =
+        FILTERS
+            .chain()
+            .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_LOCK))
+            .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
+            .filter(FILTERS.value().exactMatch(uuid));
+    Mutation deleteCell =
+        Mutation.create()
+            .deleteCells(MetadataTableAdminDao.CF_LOCK, MetadataTableAdminDao.QUALIFIER_DEFAULT)
+            .setCell(
+                MetadataTableAdminDao.CF_SHOULD_DELETE, MetadataTableAdminDao.QUALIFIER_DEFAULT, 1);
+    ConditionalRowMutation rowMutation =
+        ConditionalRowMutation.create(tableId, rowKey).condition(lockCellFilter).then(deleteCell);
+    return dataClient.checkAndMutateRow(rowMutation);
   }
 
   /**
-   * Delete the row.
+   * This is the 2nd step of 2 phase delete of StreamPartition. Delete the row key represented by
+   * the partition. This represents that the partition will no longer be streamed. Only delete if
+   * shouldDelete bit is set.
    *
-   * @param rowKey row key of the row to delete
+   * @param partition forms the row key of the row to delete
    */
-  public void deleteRowKey(ByteString rowKey) {
-    RowMutation rowMutation = RowMutation.create(tableId, rowKey).deleteRow();
-    dataClient.mutateRow(rowMutation);
+  public boolean deleteStreamPartitionRow(ByteStringRange partition) {
+    LOG.debug("Delete metadata row");
+    ByteString rowKey = convertPartitionToStreamPartitionRowKey(partition);
+    Filter shouldDeleteFilter =
+        FILTERS
+            .chain()
+            .filter(FILTERS.family().exactMatch(MetadataTableAdminDao.CF_SHOULD_DELETE))
+            .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
+            .filter(FILTERS.value().exactMatch(ByteString.copyFrom(Longs.toByteArray(1))));
+    Mutation deleteRow = Mutation.create().deleteRow();
+    ConditionalRowMutation rowMutation =
+        ConditionalRowMutation.create(tableId, rowKey)
+            .condition(shouldDeleteFilter)
+            .then(deleteRow);
+    return dataClient.checkAndMutateRow(rowMutation);
   }
 
   /**
-   * Lock the partition in the metadata table for the DoFn streaming it. Only one DoFn is allowed to
-   * stream a specific partition at any time. Each DoFn has an uuid and will try to lock the
-   * partition at the very start of the stream. If another DoFn has already locked the partition
-   * (i.e. the uuid in the cell for the partition belongs to the DoFn), any future DoFn trying to
-   * lock the same partition will and terminate.
+   * Return true if the uuid holds the lock of the partition.
    *
-   * @param partition form the row key in the metadata table to lock
-   * @param uuid id of the DoFn
+   * @param partition partition to check if lock is held
+   * @param uuid to check if it holds the lock
    * @return true if uuid holds the lock, otherwise false.
    */
-  public boolean lockPartition(ByteStringRange partition, String uuid) {
-    LOG.debug("Locking partition before processing stream");
-
+  public boolean doHoldLock(ByteStringRange partition, String uuid) {
     ByteString rowKey = convertPartitionToStreamPartitionRowKey(partition);
     Filter lockCellFilter =
         FILTERS
@@ -315,18 +644,53 @@ public class MetadataTableDao {
             .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
             .filter(FILTERS.limit().cellsPerRow(1));
     Row row = dataClient.readRow(tableId, rowKey, lockCellFilter);
-
-    // If the query returns non-null row, that means the lock is being held. Check if the owner is
-    // same as uuid.
     if (row != null) {
       return row.getCells().get(0).getValue().toStringUtf8().equals(uuid);
     }
+    return false;
+  }
 
-    // We cannot check whether a cell is empty, We can check if the cell matches any value. If it
-    // does not, we perform the mutation to set the cell.
+  /**
+   * Lock the partition in the metadata table for the DoFn streaming it. Only one DoFn is allowed to
+   * stream a specific partition at any time. Each DoFn has an uuid and will try to lock the
+   * partition at the very start of the stream. If another DoFn has already locked the partition
+   * (i.e. the uuid in the cell for the partition belongs to the DoFn), any future DoFn trying to
+   * lock the same partition will fail. Also unset the deletion bit.
+   *
+   * @param partitionRecord partition to lock
+   * @return true if uuid holds or acquired the lock, otherwise false.
+   */
+  public boolean lockAndRecordPartition(PartitionRecord partitionRecord) {
+    if (doHoldLock(partitionRecord.getPartition(), partitionRecord.getUuid())) {
+      return true;
+    }
+
+    // Record all the initial metadata.
     Mutation mutation =
         Mutation.create()
-            .setCell(MetadataTableAdminDao.CF_LOCK, MetadataTableAdminDao.QUALIFIER_DEFAULT, uuid);
+            .setCell(
+                MetadataTableAdminDao.CF_LOCK,
+                MetadataTableAdminDao.QUALIFIER_DEFAULT,
+                partitionRecord.getUuid())
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK,
+                MetadataTableAdminDao.QUALIFIER_DEFAULT,
+                partitionRecord.getParentLowWatermark().getMillis())
+            .deleteCells(
+                MetadataTableAdminDao.CF_SHOULD_DELETE, MetadataTableAdminDao.QUALIFIER_DEFAULT);
+    List<ChangeStreamContinuationToken> tokens =
+        partitionRecord.getChangeStreamContinuationTokens();
+    if (tokens != null) {
+      for (ChangeStreamContinuationToken token : tokens) {
+        mutation.setCell(
+            MetadataTableAdminDao.CF_INITIAL_TOKEN,
+            ByteStringRange.serializeToByteString(token.getPartition()),
+            token.toByteString());
+      }
+    }
+
+    // We cannot check whether a cell is empty, We can check if the cell matches any value. If it
+    // does not, we perform the mutation to set the cell.
     Filter matchAnyString =
         FILTERS
             .chain()
@@ -334,7 +698,8 @@ public class MetadataTableDao {
             .filter(FILTERS.qualifier().exactMatch(MetadataTableAdminDao.QUALIFIER_DEFAULT))
             .filter(FILTERS.value().regex("\\C*"));
     ConditionalRowMutation rowMutation =
-        ConditionalRowMutation.create(tableId, rowKey)
+        ConditionalRowMutation.create(
+                tableId, convertPartitionToStreamPartitionRowKey(partitionRecord.getPartition()))
             .condition(matchAnyString)
             .otherwise(mutation);
     return !dataClient.checkAndMutateRow(rowMutation);
@@ -360,8 +725,8 @@ public class MetadataTableDao {
    *
    * @return deserialized missing partitions and duration.
    */
-  public HashMap<ByteStringRange, Long> readDetectNewPartitionMissingPartitions() {
-    @Nonnull HashMap<ByteStringRange, Long> missingPartitions = new HashMap<>();
+  public HashMap<ByteStringRange, Instant> readDetectNewPartitionMissingPartitions() {
+    @Nonnull HashMap<ByteStringRange, Instant> missingPartitions = new HashMap<>();
     Filter missingPartitionsFilter =
         FILTERS
             .chain()
@@ -397,7 +762,7 @@ public class MetadataTableDao {
    * @param missingPartitionDurations missing partitions and duration.
    */
   public void writeDetectNewPartitionMissingPartitions(
-      HashMap<ByteStringRange, Long> missingPartitionDurations) {
+      HashMap<ByteStringRange, Instant> missingPartitionDurations) {
     byte[] serializedMissingPartition = SerializationUtils.serialize(missingPartitionDurations);
     RowMutation rowMutation =
         RowMutation.create(tableId, getFullDetectNewPartition())
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/DetectNewPartitionsDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/DetectNewPartitionsDoFn.java
index 5433ee86fc4..1ec1abb323a 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/DetectNewPartitionsDoFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/DetectNewPartitionsDoFn.java
@@ -24,10 +24,14 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ActionFactory;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.DetectNewPartitionsAction;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.GenerateInitialPartitionsAction;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ProcessNewPartitionsAction;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ResumeFromPreviousPipelineAction;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.DetectNewPartitionsTracker;
 import org.apache.beam.sdk.io.range.OffsetRange;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.DoFn.UnboundedPerElement;
@@ -41,7 +45,7 @@ import org.joda.time.Instant;
 @SuppressWarnings("initialization.fields.uninitialized")
 @Internal
 @UnboundedPerElement
-public class DetectNewPartitionsDoFn extends DoFn<Instant, PartitionRecord> {
+public class DetectNewPartitionsDoFn extends DoFn<InitialPipelineState, PartitionRecord> {
   private static final long serialVersionUID = 8052524268978107367L;
   @Nullable private final Instant endTime;
 
@@ -62,8 +66,9 @@ public class DetectNewPartitionsDoFn extends DoFn<Instant, PartitionRecord> {
   }
 
   @GetInitialWatermarkEstimatorState
-  public Instant getInitialWatermarkEstimatorState(@Element Instant startTime) {
-    return startTime;
+  public Instant getInitialWatermarkEstimatorState(
+      @Element InitialPipelineState initialPipelineState) {
+    return initialPipelineState.getStartTime();
   }
 
   @NewWatermarkEstimator
@@ -79,7 +84,7 @@ public class DetectNewPartitionsDoFn extends DoFn<Instant, PartitionRecord> {
 
   @NewTracker
   public OffsetRangeTracker restrictionTracker(@Restriction OffsetRange restriction) {
-    return new OffsetRangeTracker(restriction);
+    return new DetectNewPartitionsTracker(restriction.getFrom());
   }
 
   // We never want to scale based on this DoFn, so we return a constant backlog estimate of zero.
@@ -92,22 +97,31 @@ public class DetectNewPartitionsDoFn extends DoFn<Instant, PartitionRecord> {
   public void setup() throws IOException {
     final MetadataTableDao metadataTableDao = daoFactory.getMetadataTableDao();
     final ChangeStreamDao changeStreamDao = daoFactory.getChangeStreamDao();
+    ProcessNewPartitionsAction processNewPartitionsAction =
+        actionFactory.processNewPartitionsAction(metrics, metadataTableDao, endTime);
     GenerateInitialPartitionsAction generateInitialPartitionsAction =
         actionFactory.generateInitialPartitionsAction(metrics, changeStreamDao, endTime);
+    ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction =
+        actionFactory.resumeFromPreviousPipelineAction(
+            metrics, metadataTableDao, endTime, processNewPartitionsAction);
     detectNewPartitionsAction =
         actionFactory.detectNewPartitionsAction(
-            metrics, metadataTableDao, endTime, generateInitialPartitionsAction);
+            metrics,
+            metadataTableDao,
+            endTime,
+            generateInitialPartitionsAction,
+            resumeFromPreviousPipelineAction,
+            processNewPartitionsAction);
   }
 
   @ProcessElement
   public ProcessContinuation processElement(
-      @Element Instant startTime,
+      @Element InitialPipelineState initialPipelineState,
       RestrictionTracker<OffsetRange, Long> tracker,
       OutputReceiver<PartitionRecord> receiver,
-      ManualWatermarkEstimator<Instant> watermarkEstimator,
-      BundleFinalizer bundleFinalizer)
+      ManualWatermarkEstimator<Instant> watermarkEstimator)
       throws Exception {
     return detectNewPartitionsAction.run(
-        tracker, receiver, watermarkEstimator, bundleFinalizer, startTime);
+        tracker, receiver, watermarkEstimator, initialPipelineState);
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java
index 51ef62bcd4e..07826ee1db7 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFn.java
@@ -20,7 +20,10 @@ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn;
 import java.io.IOException;
 import java.io.Serializable;
 import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO.ExistingPipelineOptions;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.DetectNewPartitionsState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.joda.time.Instant;
 import org.slf4j.Logger;
@@ -31,23 +34,28 @@ import org.slf4j.LoggerFactory;
  * pipeline.
  */
 @Internal
-public class InitializeDoFn extends DoFn<byte[], Instant> implements Serializable {
+public class InitializeDoFn extends DoFn<byte[], InitialPipelineState> implements Serializable {
   private static final long serialVersionUID = 1868189906451252363L;
 
   private static final Logger LOG = LoggerFactory.getLogger(InitializeDoFn.class);
   private final DaoFactory daoFactory;
   private final String metadataTableAppProfileId;
   private Instant startTime;
+  private final ExistingPipelineOptions existingPipelineOptions;
 
   public InitializeDoFn(
-      DaoFactory daoFactory, String metadataTableAppProfileId, Instant startTime) {
+      DaoFactory daoFactory,
+      String metadataTableAppProfileId,
+      Instant startTime,
+      ExistingPipelineOptions existingPipelineOptions) {
     this.daoFactory = daoFactory;
     this.metadataTableAppProfileId = metadataTableAppProfileId;
     this.startTime = startTime;
+    this.existingPipelineOptions = existingPipelineOptions;
   }
 
   @ProcessElement
-  public void processElement(OutputReceiver<Instant> receiver) throws IOException {
+  public void processElement(OutputReceiver<InitialPipelineState> receiver) throws IOException {
     LOG.info(daoFactory.getStreamTableDebugString());
     LOG.info(daoFactory.getMetadataTableDebugString());
     LOG.info("ChangeStreamName: " + daoFactory.getChangeStreamName());
@@ -69,8 +77,64 @@ public class InitializeDoFn extends DoFn<byte[], Instant> implements Serializabl
           "Reusing existing metadata table: " + daoFactory.getMetadataTableAdminDao().getTableId());
     }
 
-    daoFactory.getMetadataTableDao().writeDetectNewPartitionVersion();
+    boolean resume = false;
+    DetectNewPartitionsState detectNewPartitionsState =
+        daoFactory.getMetadataTableDao().readDetectNewPartitionsState();
 
-    receiver.output(startTime);
+    switch (existingPipelineOptions) {
+      case NEW:
+        // clean up table
+        LOG.info(
+            "Cleaning up an old pipeline with the same change stream name to start a new pipeline with the same name.");
+        daoFactory.getMetadataTableAdminDao().cleanUpPrefix();
+        break;
+      case RESUME_OR_NEW:
+        // perform resumption.
+        if (detectNewPartitionsState != null) {
+          resume = true;
+          startTime = detectNewPartitionsState.getWatermark();
+          LOG.info("Resuming from previous pipeline with low watermark of {}", startTime);
+        } else {
+          LOG.info(
+              "Attempted to resume, but previous watermark does not exist, starting at {}",
+              startTime);
+          daoFactory.getMetadataTableAdminDao().cleanUpPrefix();
+        }
+        break;
+      case RESUME_OR_FAIL:
+        // perform resumption.
+        if (detectNewPartitionsState != null) {
+          resume = true;
+          startTime = detectNewPartitionsState.getWatermark();
+          LOG.info("Resuming from previous pipeline with low watermark of {}", startTime);
+        } else {
+          LOG.error("Previous pipeline with the same change stream name doesn't exist, stopping");
+          return;
+        }
+        break;
+      case FAIL_IF_EXISTS:
+        if (detectNewPartitionsState != null) {
+          LOG.error(
+              "A previous pipeline exists with the same change stream name and existingPipelineOption is set to FAIL_IF_EXISTS.");
+          return;
+        }
+        // We still want to clean up any existing prefixes in case there are lingering metadata that
+        // would interfere with the new run.
+        daoFactory.getMetadataTableAdminDao().cleanUpPrefix();
+        break;
+      case SKIP_CLEANUP:
+        if (detectNewPartitionsState != null) {
+          LOG.error(
+              "A previous pipeline exists with the same change stream name and existingPipelineOption is set to SKIP_CLEANUP. This option should only be used in tests.");
+          return;
+        }
+        break;
+      default:
+        LOG.error("Unexpected existingPipelineOptions option.");
+        // terminate pipeline
+        return;
+    }
+    daoFactory.getMetadataTableDao().writeDetectNewPartitionVersion();
+    receiver.output(new InitialPipelineState(startTime, resume));
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java
index cbd2895efe1..03cae41c5bd 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/encoder/MetadataTableEncoder.java
@@ -17,8 +17,13 @@
  */
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder;
 
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.microsecondToInstant;
+
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
 import com.google.cloud.bigtable.data.v2.models.Row;
 import com.google.cloud.bigtable.data.v2.models.RowCell;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.util.ArrayList;
 import java.util.List;
 import org.apache.beam.sdk.annotations.Internal;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
@@ -44,13 +49,28 @@ public class MetadataTableEncoder {
     return Instant.ofEpochMilli(Longs.fromByteArray(cells.get(0).getValue().toByteArray()));
   }
 
+  /**
+   * Return the timestamp (the time it was updated) of the watermark cell.
+   *
+   * @param row row to extract the timestamp from
+   * @return the timestamp of the watermark cell.
+   */
+  public static @Nullable Instant parseWatermarkLastUpdatedFromRow(Row row) {
+    List<RowCell> cells =
+        row.getCells(MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT);
+    if (cells.size() == 0) {
+      return null;
+    }
+    return microsecondToInstant(cells.get(0).getTimestamp());
+  }
+
   /**
    * Read the continuation token cell of a row from ReadRows.
    *
    * @param row to extract the token from
    * @return the token of the row
    */
-  public static @Nullable String getTokenFromRow(Row row) {
+  public static @Nullable String parseTokenFromRow(Row row) {
     List<RowCell> cells =
         row.getCells(
             MetadataTableAdminDao.CF_CONTINUATION_TOKEN, MetadataTableAdminDao.QUALIFIER_DEFAULT);
@@ -59,4 +79,48 @@ public class MetadataTableEncoder {
     }
     return cells.get(0).getValue().toStringUtf8();
   }
+
+  /**
+   * Returns true if row is locked.
+   *
+   * @param row to evaluate if it's locked
+   * @return true if row is locked. Otherwise, false.
+   */
+  public static boolean isRowLocked(Row row) {
+    return row.getCells(MetadataTableAdminDao.CF_LOCK, MetadataTableAdminDao.QUALIFIER_DEFAULT)
+            .size()
+        != 0;
+  }
+
+  /**
+   * Returns the uuid from a row.
+   *
+   * @param row to extract the uuid
+   * @return the uuid of the row
+   */
+  public static @Nullable String parseLockUuid(Row row) {
+    List<RowCell> cells =
+        row.getCells(MetadataTableAdminDao.CF_LOCK, MetadataTableAdminDao.QUALIFIER_DEFAULT);
+    if (cells.size() == 0) {
+      return null;
+    }
+    return cells.get(0).getValue().toStringUtf8();
+  }
+
+  /**
+   * Return a list of initial token from a row.
+   *
+   * @param row to extract the initial tokens
+   * @return a list of tokens from the row
+   * @throws InvalidProtocolBufferException when failed to parse tokens.
+   */
+  public static List<ChangeStreamContinuationToken> parseInitialContinuationTokens(Row row)
+      throws InvalidProtocolBufferException {
+    List<ChangeStreamContinuationToken> tokens = new ArrayList<>();
+    List<RowCell> cells = row.getCells(MetadataTableAdminDao.CF_INITIAL_TOKEN);
+    for (RowCell rowCell : cells) {
+      tokens.add(ChangeStreamContinuationToken.fromByteString(rowCell.getValue()));
+    }
+    return tokens;
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/DetectNewPartitionsState.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/DetectNewPartitionsState.java
new file mode 100644
index 00000000000..2e43124d56c
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/DetectNewPartitionsState.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+import org.apache.beam.sdk.annotations.Internal;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.Instant;
+
+/**
+ * Metadata of the progress of {@link
+ * org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.DetectNewPartitionsDoFn} from the metadata
+ * table.
+ */
+@Internal
+public class DetectNewPartitionsState implements Serializable {
+  private static final long serialVersionUID = 6587699311321738060L;
+
+  private final Instant watermark;
+  private final Instant watermarkLastUpdated;
+
+  public DetectNewPartitionsState(Instant watermark, Instant watermarkLastUpdated) {
+    this.watermark = watermark;
+    this.watermarkLastUpdated = watermarkLastUpdated;
+  }
+
+  public Instant getWatermark() {
+    return watermark;
+  }
+
+  public Instant getWatermarkLastUpdated() {
+    return watermarkLastUpdated;
+  }
+
+  @Override
+  public boolean equals(@Nullable Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof DetectNewPartitionsState)) {
+      return false;
+    }
+    DetectNewPartitionsState that = (DetectNewPartitionsState) o;
+    return Objects.equals(getWatermark(), that.getWatermark())
+        && Objects.equals(getWatermarkLastUpdated(), that.getWatermarkLastUpdated());
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(getWatermark(), getWatermarkLastUpdated());
+  }
+
+  @Override
+  public String toString() {
+    return "DetectNewPartitions{"
+        + "watermark="
+        + watermark
+        + ", watermarkLastUpdated="
+        + watermarkLastUpdated
+        + '}';
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/InitialPipelineState.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/InitialPipelineState.java
new file mode 100644
index 00000000000..3f0da38a128
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/InitialPipelineState.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.model;
+
+import java.io.Serializable;
+import java.util.Objects;
+import org.apache.beam.sdk.annotations.Internal;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.joda.time.Instant;
+
+/**
+ * States to initialize a pipeline outputted by {@link
+ * org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn.InitializeDoFn}.
+ */
+@Internal
+public class InitialPipelineState implements Serializable {
+  private static final long serialVersionUID = 7685843906645495071L;
+  private final Instant startTime;
+  private final boolean resume;
+
+  public InitialPipelineState(Instant startTime, boolean resume) {
+    this.startTime = startTime;
+    this.resume = resume;
+  }
+
+  public Instant getStartTime() {
+    return startTime;
+  }
+
+  public boolean isResume() {
+    return resume;
+  }
+
+  @Override
+  public boolean equals(@Nullable Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof InitialPipelineState)) {
+      return false;
+    }
+    InitialPipelineState that = (InitialPipelineState) o;
+    return isResume() == that.isResume() && Objects.equals(getStartTime(), that.getStartTime());
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(getStartTime(), isResume());
+  }
+
+  @Override
+  public String toString() {
+    return "InitialPipeline{" + "startTime=" + startTime + ", resume=" + resume + '}';
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/NewPartition.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/NewPartition.java
new file mode 100644
index 00000000000..08508172ad4
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/NewPartition.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.model;
+
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
+
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
+import org.joda.time.Instant;
+
+/**
+ * Represent new partition as a result of splits and merges.
+ *
+ * <p>Parent partitions stop streaming and record their change stream continuation tokens. The
+ * tokens are collected to continue the stream from the new partition.
+ */
+@Internal
+public class NewPartition implements Serializable {
+  private static final long serialVersionUID = -6530725494713332884L;
+  ByteStringRange partition;
+  List<ChangeStreamContinuationToken> changeStreamContinuationTokens;
+  Instant lowWatermark;
+  Instant lastUpdated;
+
+  public NewPartition(
+      ByteStringRange partition,
+      List<ChangeStreamContinuationToken> changeStreamContinuationTokens,
+      Instant lowWatermark,
+      Instant lastUpdated) {
+    this.partition = partition;
+    this.changeStreamContinuationTokens = changeStreamContinuationTokens;
+    this.lowWatermark = lowWatermark;
+    this.lastUpdated = lastUpdated;
+  }
+
+  public NewPartition(
+      ByteStringRange partition,
+      List<ChangeStreamContinuationToken> changeStreamContinuationTokens,
+      Instant lowWatermark) {
+    this(partition, changeStreamContinuationTokens, lowWatermark, Instant.EPOCH);
+  }
+
+  public ByteStringRange getPartition() {
+    return partition;
+  }
+
+  public List<ChangeStreamContinuationToken> getChangeStreamContinuationTokens() {
+    return changeStreamContinuationTokens;
+  }
+
+  public Instant getLowWatermark() {
+    return lowWatermark;
+  }
+
+  public List<ByteStringRange> getParentPartitions() {
+    return getChangeStreamContinuationTokens().stream()
+        .map(ChangeStreamContinuationToken::getPartition)
+        .collect(Collectors.toList());
+  }
+
+  public Instant getLastUpdated() {
+    return lastUpdated;
+  }
+
+  /**
+   * Return a new NewPartition that only contains one token that matches the parentPartition. If the
+   * parentPartition does not exist, return null.
+   *
+   * @param parentPartition to parent partition to find within the tokens.
+   * @return a NewPartition with one token with parentPartition. null otherwise.
+   */
+  public @Nullable NewPartition getSingleTokenNewPartition(ByteStringRange parentPartition) {
+    for (ChangeStreamContinuationToken parentToken : getChangeStreamContinuationTokens()) {
+      if (parentToken.getPartition().equals(parentPartition)) {
+        // The partition stays the same for the splices because that's the row key to find the cells
+        // to clean up.
+        return new NewPartition(
+            getPartition(),
+            Collections.singletonList(parentToken),
+            getLowWatermark(),
+            getLastUpdated());
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public boolean equals(@Nullable Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof NewPartition)) {
+      return false;
+    }
+    NewPartition that = (NewPartition) o;
+    return Objects.equals(getPartition(), that.getPartition())
+        && Objects.equals(
+            getChangeStreamContinuationTokens(), that.getChangeStreamContinuationTokens())
+        && Objects.equals(getLowWatermark(), that.getLowWatermark())
+        && Objects.equals(getLastUpdated(), that.getLastUpdated());
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(
+        getPartition(), getChangeStreamContinuationTokens(), getLowWatermark(), getLastUpdated());
+  }
+
+  @Override
+  public String toString() {
+    return "NewPartition{"
+        + "partition="
+        + formatByteStringRange(partition)
+        + ", changeStreamContinuationTokens="
+        + changeStreamContinuationTokens.stream()
+            .map(t -> formatByteStringRange(t.getPartition()) + " => {" + t.getToken() + "}")
+            .collect(Collectors.joining(", ", "{", "}"))
+        + ", lowWatermark="
+        + lowWatermark
+        + ", lastUpdated="
+        + lastUpdated
+        + '}';
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/PartitionRecord.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/PartitionRecord.java
index 82cd9b566a7..faf006b94e4 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/PartitionRecord.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/PartitionRecord.java
@@ -35,35 +35,45 @@ import org.joda.time.Instant;
  */
 @Internal
 public class PartitionRecord implements Serializable {
-  private static final long serialVersionUID = -7613861834142734474L;
+  private static final long serialVersionUID = -4524061648930484599L;
 
-  private ByteStringRange partition;
+  private final ByteStringRange partition;
   @Nullable private Instant startTime;
   @Nullable private List<ChangeStreamContinuationToken> changeStreamContinuationTokens;
   @Nullable private Instant endTime;
   private String uuid;
-  private Instant parentLowWatermark;
+  private final Instant parentLowWatermark;
+  private final List<NewPartition> parentPartitions;
+
+  public PartitionRecord(
+      ByteStringRange partition,
+      List<ChangeStreamContinuationToken> changeStreamContinuationTokens,
+      Instant parentLowWatermark,
+      List<NewPartition> parentPartitions) {
+    this(partition, changeStreamContinuationTokens, "", parentLowWatermark, parentPartitions, null);
+  }
+
+  public PartitionRecord(
+      ByteStringRange partition,
+      Instant startTime,
+      Instant parentLowWatermark,
+      List<NewPartition> parentPartitions) {
+    this(partition, startTime, "", parentLowWatermark, parentPartitions, null);
+  }
 
   public PartitionRecord(
       ByteStringRange partition,
       Instant startTime,
       String uuid,
       Instant parentLowWatermark,
+      List<NewPartition> parentPartitions,
       @Nullable Instant endTime) {
     this.partition = partition;
     this.startTime = startTime;
     this.uuid = uuid;
     this.parentLowWatermark = parentLowWatermark;
     this.endTime = endTime;
-  }
-
-  @Nullable
-  public Instant getEndTime() {
-    return endTime;
-  }
-
-  public void setEndTime(@Nullable Instant endTime) {
-    this.endTime = endTime;
+    this.parentPartitions = parentPartitions;
   }
 
   public PartitionRecord(
@@ -71,12 +81,14 @@ public class PartitionRecord implements Serializable {
       List<ChangeStreamContinuationToken> changeStreamContinuationTokens,
       String uuid,
       Instant parentLowWatermark,
+      List<NewPartition> parentPartitions,
       @Nullable Instant endTime) {
     this.partition = partition;
     this.changeStreamContinuationTokens = changeStreamContinuationTokens;
     this.uuid = uuid;
     this.parentLowWatermark = parentLowWatermark;
     this.endTime = endTime;
+    this.parentPartitions = parentPartitions;
   }
 
   @Nullable
@@ -84,42 +96,38 @@ public class PartitionRecord implements Serializable {
     return startTime;
   }
 
-  public void setStartTime(@Nullable Instant startTime) {
-    this.startTime = startTime;
-  }
-
   public String getUuid() {
     return uuid;
   }
 
-  public void setUuid(String uuid) {
-    this.uuid = uuid;
-  }
-
   public Instant getParentLowWatermark() {
     return parentLowWatermark;
   }
 
-  public void setParentLowWatermark(Instant parentLowWatermark) {
-    this.parentLowWatermark = parentLowWatermark;
-  }
-
   public ByteStringRange getPartition() {
     return partition;
   }
 
-  public void setPartition(ByteStringRange partition) {
-    this.partition = partition;
-  }
-
   @Nullable
   public List<ChangeStreamContinuationToken> getChangeStreamContinuationTokens() {
     return changeStreamContinuationTokens;
   }
 
-  public void setChangeStreamContinuationTokens(
-      @Nullable List<ChangeStreamContinuationToken> changeStreamContinuationTokens) {
-    this.changeStreamContinuationTokens = changeStreamContinuationTokens;
+  @Nullable
+  public Instant getEndTime() {
+    return endTime;
+  }
+
+  public List<NewPartition> getParentPartitions() {
+    return parentPartitions;
+  }
+
+  public void setUuid(String uuid) {
+    this.uuid = uuid;
+  }
+
+  public void setEndTime(@Nullable Instant endTime) {
+    this.endTime = endTime;
   }
 
   @Override
@@ -131,13 +139,14 @@ public class PartitionRecord implements Serializable {
       return false;
     }
     PartitionRecord that = (PartitionRecord) o;
-    return getPartition().equals(that.getPartition())
+    return Objects.equals(getPartition(), that.getPartition())
         && Objects.equals(getStartTime(), that.getStartTime())
         && Objects.equals(
             getChangeStreamContinuationTokens(), that.getChangeStreamContinuationTokens())
         && Objects.equals(getEndTime(), that.getEndTime())
-        && getUuid().equals(that.getUuid())
-        && Objects.equals(getParentLowWatermark(), that.getParentLowWatermark());
+        && Objects.equals(getUuid(), that.getUuid())
+        && Objects.equals(getParentLowWatermark(), that.getParentLowWatermark())
+        && Objects.equals(parentPartitions, that.parentPartitions);
   }
 
   @Override
@@ -148,7 +157,8 @@ public class PartitionRecord implements Serializable {
         getChangeStreamContinuationTokens(),
         getEndTime(),
         getUuid(),
-        getParentLowWatermark());
+        getParentLowWatermark(),
+        parentPartitions);
   }
 
   @Override
@@ -160,11 +170,15 @@ public class PartitionRecord implements Serializable {
         + startTime
         + ", changeStreamContinuationTokens="
         + changeStreamContinuationTokens
+        + ", endTime="
+        + endTime
         + ", uuid='"
         + uuid
         + '\''
         + ", parentLowWatermark="
         + parentLowWatermark
+        + ", parentPartitions="
+        + parentPartitions
         + '}';
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/StreamPartitionWithWatermark.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/StreamPartitionWithWatermark.java
new file mode 100644
index 00000000000..7562d9b5dd4
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/model/StreamPartitionWithWatermark.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.model;
+
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
+
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.io.Serializable;
+import java.util.Objects;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
+import org.joda.time.Instant;
+
+@Internal
+public class StreamPartitionWithWatermark implements Serializable {
+  private static final long serialVersionUID = -318960470803696894L;
+
+  final ByteStringRange partition;
+  final Instant watermark;
+
+  public StreamPartitionWithWatermark(ByteStringRange partition, Instant watermark) {
+    this.partition = partition;
+    this.watermark = watermark;
+  }
+
+  public ByteStringRange getPartition() {
+    return partition;
+  }
+
+  public Instant getWatermark() {
+    return watermark;
+  }
+
+  @Override
+  public boolean equals(@Nullable Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof StreamPartitionWithWatermark)) {
+      return false;
+    }
+    StreamPartitionWithWatermark that = (StreamPartitionWithWatermark) o;
+    return Objects.equals(getPartition(), that.getPartition())
+        && Objects.equals(getWatermark(), that.getWatermark());
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(getPartition(), getWatermark());
+  }
+
+  @Override
+  public String toString() {
+    return "StreamPartitionWithWatermark{"
+        + "partition="
+        + formatByteStringRange(partition)
+        + ", watermark="
+        + watermark
+        + '}';
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleaner.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleaner.java
new file mode 100644
index 00000000000..cd45306eee9
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleaner.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler;
+
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.doPartitionsOverlap;
+
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.util.ArrayList;
+import java.util.List;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.joda.time.Duration;
+
+@Internal
+public class OrphanedMetadataCleaner {
+  private static final Duration ORPHANED_NEW_PARTITION_WAIT = Duration.standardMinutes(15);
+  List<NewPartition> newPartitions = new ArrayList<>();
+  @Nullable List<ByteStringRange> missingPartitions = null;
+
+  /**
+   * Add NewPartition if it hasn't been updated for 15 minutes.
+   *
+   * @param newPartition new partition to clean up.
+   */
+  public void addIncompleteNewPartitions(NewPartition newPartition) {
+    if (newPartition.getLastUpdated().plus(ORPHANED_NEW_PARTITION_WAIT).isBeforeNow()) {
+      this.newPartitions.add(newPartition);
+    }
+  }
+
+  /**
+   * Add all the missingPartitions. This must be called in order evaluate orphaned new partition.
+   *
+   * @param missingPartitions add missingPartitions.
+   */
+  public void addMissingPartitions(List<ByteStringRange> missingPartitions) {
+    if (this.missingPartitions == null) {
+      this.missingPartitions = new ArrayList<>();
+    }
+    this.missingPartitions.addAll(missingPartitions);
+  }
+
+  /**
+   * Returns a list of NewPartition that have been around for a while and do not overlap with any
+   * missing partition.
+   *
+   * <p>Must call <code>addMissingPartitions</code> before otherwise no orphan can be cleaned up.
+   *
+   * @return NewPartitions safe to clean up.
+   */
+  public List<NewPartition> getOrphanedNewPartitions() {
+    List<NewPartition> orphanedNewPartitions = new ArrayList<>();
+    List<ByteStringRange> missingPartitions = this.missingPartitions;
+    if (missingPartitions == null) {
+      return orphanedNewPartitions;
+    }
+    for (NewPartition newPartition : this.newPartitions) {
+      boolean hasOverlap = false;
+      for (ByteStringRange missingPartition : missingPartitions) {
+        if (doPartitionsOverlap(newPartition.getPartition(), missingPartition)) {
+          hasOverlap = true;
+          break;
+        }
+      }
+      if (!hasOverlap) {
+        orphanedNewPartitions.add(newPartition);
+      }
+    }
+    return orphanedNewPartitions;
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java
index 931e8a6d88f..41e07081d69 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconciler.java
@@ -17,18 +17,26 @@
  */
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler;
 
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.coverSameKeySpace;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.doPartitionsOverlap;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.isValidPartition;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingPartitionsFrom;
 
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
 import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
-import com.google.protobuf.ByteString;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
+import java.util.Map;
 import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.joda.time.Duration;
 import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * There can be a race when many splits and merges happen to a single partition in quick succession.
@@ -58,15 +66,20 @@ import org.joda.time.Instant;
  */
 @Internal
 public class PartitionReconciler {
-  HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile = new HashMap<>();
-  HashMap<ByteStringRange, ByteString> newPartitions = new HashMap<>();
-  MetadataTableDao metadataTableDao;
+  private static final Logger LOG = LoggerFactory.getLogger(PartitionReconciler.class);
+
+  private HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+  private final List<NewPartition> newPartitions = new ArrayList<>();
+  private final MetadataTableDao metadataTableDao;
+  private final ChangeStreamMetrics metrics;
 
   // The amount of delay allowed before we consider a partition to be probably missing.
-  private static final long MISSING_PARTITION_DELAY_MILLI = 5 * 60 * 1000L;
+  private static final Duration MISSING_PARTITION_SHORT_DELAY = Duration.standardMinutes(1);
+  private static final Duration MISSING_PARTITION_LONG_DELAY = Duration.standardMinutes(10);
 
-  public PartitionReconciler(MetadataTableDao metadataTableDao) {
+  public PartitionReconciler(MetadataTableDao metadataTableDao, ChangeStreamMetrics metrics) {
     this.metadataTableDao = metadataTableDao;
+    this.metrics = metrics;
   }
 
   /**
@@ -87,67 +100,143 @@ public class PartitionReconciler {
    * @param missingPartitions partitions not being streamed.
    */
   public void addMissingPartitions(List<ByteStringRange> missingPartitions) {
-    HashMap<ByteStringRange, Long> alreadyMissingPartitions =
+    HashMap<ByteStringRange, Instant> alreadyMissingPartitionDurations =
         metadataTableDao.readDetectNewPartitionMissingPartitions();
-    HashMap<ByteStringRange, Long> missingPartitionDuration = new HashMap<>();
-    long now = Instant.now().getMillis();
+    missingPartitionDurations = new HashMap<>();
+    Instant now = Instant.now();
 
     for (ByteStringRange missingPartition : missingPartitions) {
-      if (!isValidPartition(missingPartition)) {
-        continue;
-      }
-      if (alreadyMissingPartitions.containsKey(missingPartition)) {
-        missingPartitionDuration.put(
-            missingPartition, alreadyMissingPartitions.get(missingPartition));
-        if (alreadyMissingPartitions.get(missingPartition) + MISSING_PARTITION_DELAY_MILLI < now) {
-          partitionsToReconcile.put(missingPartition, new HashSet<>());
-        }
-      } else {
-        missingPartitionDuration.put(missingPartition, now);
-      }
+      missingPartitionDurations.put(
+          missingPartition, alreadyMissingPartitionDurations.getOrDefault(missingPartition, now));
     }
-    metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDuration);
+    metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
   }
 
   /**
-   * Capture NewPartition row that's waiting to be created. If any of these NewPartition row
+   * Capture NewPartition row that cannot merge on its own. If any of these NewPartition row
    * overlaps with partition we notice are missing and needs to be reconciled, we will need to clean
    * up these NewPartition to avoid future conflicts and inconsistencies.
    *
-   * @param partition new partitions waiting to be created.
-   * @param rowKey the full row key of the new partition.
+   * @param newPartition new partition waiting to be created.
    */
-  public void addNewPartition(ByteStringRange partition, ByteString rowKey) {
-    newPartitions.put(partition, rowKey);
+  public void addIncompleteNewPartitions(NewPartition newPartition) {
+    newPartitions.add(newPartition);
   }
 
   /**
-   * Find overlapping partitions between partitionToReconcile and newPartitions. This is to support
-   * the effort of identifying the new partition rows that are stuck and needs to be cleaned up
-   * because we have successfully reconciled the problem and created a new partition.
+   * Get a list of single token new partitions that have parent partition overlapping
+   * missingPartitions. This is to support the effort of identifying the new partition rows that are
+   * stuck and needs to be cleaned up because we have successfully reconciled the problem and
+   * created a new partition.
    *
-   * @param partitionToReconcile partition that will be created
+   * @param missingPartition partition that will be created
    * @return a set of new partitions that overlaps with partitionToReconcile
    */
-  private Set<ByteString> findOverlappingNewPartitions(ByteStringRange partitionToReconcile) {
-    Set<ByteString> overlappingRowKey = new HashSet<>();
-    for (ByteStringRange newPartition : newPartitions.keySet()) {
-      if (doPartitionsOverlap(newPartition, partitionToReconcile)) {
-        overlappingRowKey.add(newPartitions.get(newPartition));
+  private List<NewPartition> findOverlappingNewPartitions(ByteStringRange missingPartition) {
+    // TODO: Possibly precompute a map from parentPartitions to newPartitions on the first call to
+    //  this function so we don't have to repeat it.
+    List<NewPartition> overlappingNewPartitions = new ArrayList<>();
+    for (NewPartition newPartition : newPartitions) {
+      for (ByteStringRange parentPartition : newPartition.getParentPartitions()) {
+        if (doPartitionsOverlap(parentPartition, missingPartition)) {
+          // Return a NewPartition that only contains the parentPartition.
+          NewPartition splicedNewPartition =
+              newPartition.getSingleTokenNewPartition(parentPartition);
+          if (splicedNewPartition == null) {
+            continue;
+          }
+          overlappingNewPartitions.add(splicedNewPartition);
+        }
       }
     }
-    return overlappingRowKey;
+    return overlappingNewPartitions;
   }
 
   /**
-   * Match partitions that have been missing for a while and need to be reconciled with NewPartition
-   * row key. Find NewPartition row key that overlaps with the reconciled partitions to clean them
-   * up.
+   * For missing partitions, try to organize the mismatched parent tokens in a way to fill the
+   * missing partitions.
+   *
+   * <p>If there are parent tokens that when combined form a missing partition, it can be outputted
+   * as a merge of the missing partition.
+   *
+   * <p>If there are no parent tokens for a missing partition, it will need to be reconciled with
+   * adjusted low watermark. This is a catch-all solution. We don't expect to ever get into this
+   * situation. Missing partitions should all be mismatched merges that can be reconciled by
+   * organizing them correctly.
    *
-   * @return missing partitions and related NewPartition rows keys to delete.
+   * @param lowWatermark watermark that all reconciled partition should have
+   * @param startTime to help compute optimal reconcile point
+   * @return reconciled PartitionRecord.
    */
-  public HashMap<ByteStringRange, Set<ByteString>> getPartitionsToReconcile() {
-    partitionsToReconcile.replaceAll((r, v) -> findOverlappingNewPartitions(r));
-    return partitionsToReconcile;
+  public List<PartitionRecord> getPartitionsToReconcile(Instant lowWatermark, Instant startTime) {
+    // This value is calculated in case that we reconcile without continuation tokens, we will use
+    // an hour prior to low watermark because low watermark is only an estimate. By reading back 1
+    // hour, it should cover any changes missed. We also want to make sure that the reconcile time
+    // isn't before the start time of the pipeline.
+    Instant reconciledTime = lowWatermark.minus(Duration.standardMinutes(60));
+    if (reconciledTime.compareTo(startTime) < 0) {
+      reconciledTime = startTime;
+    }
+    List<PartitionRecord> reconciledPartitions = new ArrayList<>();
+    for (Map.Entry<ByteStringRange, Instant> partitionDuration :
+        missingPartitionDurations.entrySet()) {
+      // The partition hasn't been missing for even the short duration.
+      if (!partitionDuration.getValue().plus(MISSING_PARTITION_SHORT_DELAY).isBeforeNow()) {
+        continue;
+      }
+      ByteStringRange missingPartition = partitionDuration.getKey();
+      List<NewPartition> overlappingNewPartitions = findOverlappingNewPartitions(missingPartition);
+      List<ByteStringRange> overlappingParentPartitions = new ArrayList<>();
+      for (NewPartition newPartition : overlappingNewPartitions) {
+        overlappingParentPartitions.add(newPartition.getParentPartitions().get(0));
+      }
+      // If the parents are equal to the missing partition, a new partition can be formed.
+      if (coverSameKeySpace(overlappingParentPartitions, missingPartition)) {
+        List<ChangeStreamContinuationToken> allTokens = new ArrayList<>();
+        for (NewPartition newPartition : overlappingNewPartitions) {
+          allTokens.add(newPartition.getChangeStreamContinuationTokens().get(0));
+        }
+
+        metrics.incPartitionReconciledWithTokenCount();
+        PartitionRecord record =
+            new PartitionRecord(
+                missingPartition, allTokens, lowWatermark, overlappingNewPartitions);
+        reconciledPartitions.add(record);
+        continue;
+      }
+      // The parents are not equal to the missing partition. If the missing partition has been
+      // around for more than 10 minutes, it needs to be reconciled.
+      if (partitionDuration.getValue().plus(MISSING_PARTITION_LONG_DELAY).isBeforeNow()) {
+        // Try outputting the parent partitions as their own new partition, so they can get more
+        // recent merge targets.
+        for (NewPartition newPartition : overlappingNewPartitions) {
+          metrics.incPartitionReconciledWithTokenCount();
+          PartitionRecord record =
+              new PartitionRecord(
+                  newPartition.getChangeStreamContinuationTokens().get(0).getPartition(),
+                  newPartition.getChangeStreamContinuationTokens(),
+                  lowWatermark,
+                  Collections.singletonList(newPartition));
+          reconciledPartitions.add(record);
+        }
+        // Also output partition not overlapped by new partitions.
+        // Get the missing partition from parentPartitions and missingPartition.
+        List<ByteStringRange> missingPartitionsFromParents =
+            getMissingPartitionsFrom(
+                overlappingParentPartitions,
+                missingPartition.getStart(),
+                missingPartition.getEnd());
+        for (ByteStringRange missing : missingPartitionsFromParents) {
+          // This partition is truly missing. There are no tokens representing this. We restart at
+          // the calculated reconcile time.
+          metrics.incPartitionReconciledWithoutTokenCount();
+          PartitionRecord record =
+              new PartitionRecord(missing, reconciledTime, lowWatermark, Collections.emptyList());
+          reconciledPartitions.add(record);
+          LOG.error("DNP: Reconciling partition because we're missing a token {}", record);
+        }
+      }
+    }
+    return reconciledPartitions;
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTracker.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTracker.java
new file mode 100644
index 00000000000..32f619b37c4
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTracker.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction;
+
+import org.apache.beam.sdk.annotations.Internal;
+import org.apache.beam.sdk.io.range.OffsetRange;
+import org.apache.beam.sdk.transforms.splittabledofn.GrowableOffsetRangeTracker;
+import org.apache.beam.sdk.transforms.splittabledofn.SplitResult;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+@Internal
+public class DetectNewPartitionsTracker extends GrowableOffsetRangeTracker {
+
+  public DetectNewPartitionsTracker(long start) {
+    super(start, new UnboundedRangeEndEstimator());
+  }
+
+  @Override
+  // Suppress build error that prevents this from being nullable. For some reason it
+  // believes GrowableOffsetRangeTracker trySplit is NonNull even though it is nullable
+  @SuppressWarnings("override.return")
+  public @Nullable SplitResult<OffsetRange> trySplit(double fractionOfRemainder) {
+    // Refuse to split because we only ever want one active restriction for DNP
+    if (fractionOfRemainder != 0) {
+      return null;
+    }
+    return super.trySplit(fractionOfRemainder);
+  }
+
+  // DNP should always be unbounded so we hardcode Long.MAX_VALUE
+  private static class UnboundedRangeEndEstimator implements RangeEndEstimator {
+
+    @Override
+    public long estimate() {
+      return Long.MAX_VALUE;
+    }
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java
index 57aac5203d0..923f8d3e7bb 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTracker.java
@@ -62,11 +62,10 @@ public class ReadChangeStreamPartitionProgressTracker
   /**
    * This restriction tracker is for unbounded streams.
    *
-   * @return {@link
-   *     org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker.IsBounded.UNBOUNDED}
+   * @return {@link IsBounded.UNBOUNDED}
    */
   @Override
-  public RestrictionTracker.IsBounded isBounded() {
+  public IsBounded isBounded() {
     return IsBounded.UNBOUNDED;
   }
 
@@ -76,11 +75,11 @@ public class ReadChangeStreamPartitionProgressTracker
    * runner initiated checkpoint or the streamProgress contains a closeStream response and not a
    * token.
    *
-   * @throws java.lang.IllegalStateException when the restriction is not done and there is more work
-   *     to be done.
+   * @throws IllegalStateException when the restriction is not done and there is more work to be
+   *     done.
    */
   @Override
-  public void checkDone() throws java.lang.IllegalStateException {
+  public void checkDone() throws IllegalStateException {
     boolean done =
         shouldStop || streamProgress.getCloseStream() != null || streamProgress.isFailToLock();
     Preconditions.checkState(done, "There's more work to be done");
diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java
index 6ffef3790c4..ce2cf11a842 100644
--- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java
+++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/StreamProgress.java
@@ -37,7 +37,7 @@ import org.joda.time.Instant;
  */
 @Internal
 public class StreamProgress implements Serializable {
-  private static final long serialVersionUID = -8597355120329526194L;
+  private static final long serialVersionUID = -5384329262726188695L;
 
   private @Nullable ChangeStreamContinuationToken currentToken;
   private @Nullable Instant estimatedLowWatermark;
@@ -76,6 +76,10 @@ public class StreamProgress implements Serializable {
     this.failToLock = failToLock;
   }
 
+  public boolean isEmpty() {
+    return closeStream == null && currentToken == null;
+  }
+
   @Override
   public boolean equals(@Nullable Object o) {
     if (this == o) {
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelperTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelperTest.java
index 39751230176..943a748908a 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelperTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/ByteStringRangeHelperTest.java
@@ -20,7 +20,9 @@ package org.apache.beam.sdk.io.gcp.bigtable.changestreams;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.doPartitionsOverlap;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.formatByteStringRange;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getIntersectingPartition;
-import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingAndOverlappingPartitionsFromKeySpace;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingPartitionsFrom;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getMissingPartitionsFromEntireKeySpace;
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.getOverlappingPartitions;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.isValidPartition;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.ByteStringRangeHelper.partitionsToString;
 import static org.junit.Assert.assertEquals;
@@ -29,6 +31,7 @@ import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 
 import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import com.google.protobuf.ByteString;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -40,17 +43,6 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class ByteStringRangeHelperTest {
 
-  @Test
-  public void testParentIsEntireKeySpaceIsSuperSet() {
-    List<ByteStringRange> parentPartitions = new ArrayList<>();
-    ByteStringRange partition = ByteStringRange.create("", "");
-    parentPartitions.add(partition);
-
-    ByteStringRange childPartition = ByteStringRange.create("A", "B");
-
-    assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
-  }
-
   @Test
   public void testChildIsEntireKeySpaceParentIsLeftSubSet() {
     List<ByteStringRange> parentPartitions = new ArrayList<>();
@@ -58,7 +50,7 @@ public class ByteStringRangeHelperTest {
     parentPartitions.add(partition);
 
     ByteStringRange childPartition = ByteStringRange.create("", "");
-    assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
@@ -68,11 +60,11 @@ public class ByteStringRangeHelperTest {
     parentPartitions.add(partition);
 
     ByteStringRange childPartition = ByteStringRange.create("", "");
-    assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
-  public void testChildIsEntireKeySpaceParentIsSuperSet() {
+  public void testChildIsEntireKeySpaceParentCoverKeySpace() {
     List<ByteStringRange> parentPartitions = new ArrayList<>();
     ByteStringRange partition1 = ByteStringRange.create("", "n");
     ByteStringRange partition2 = ByteStringRange.create("n", "");
@@ -80,81 +72,81 @@ public class ByteStringRangeHelperTest {
     parentPartitions.add(partition2);
 
     ByteStringRange childPartition = ByteStringRange.create("", "");
-    assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertTrue(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
-  public void testParentKeySpaceStartsBeforeAndEndAfterChildIsSuperSet() {
+  public void testParentKeySpaceStartsBeforeAndEndAfterChildDoesNotCoverKeySpace() {
     List<ByteStringRange> parentPartitions = new ArrayList<>();
     ByteStringRange partition = ByteStringRange.create("A", "B");
     parentPartitions.add(partition);
 
     ByteStringRange childPartition = ByteStringRange.create("AA", "AB");
 
-    assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
-  public void testParentStartKeyIsAfterChildStartKeyIsNotSuperSet() {
+  public void testParentStartKeyIsAfterChildStartKeyDoesNotCoverKeySpace() {
     List<ByteStringRange> parentPartitions = new ArrayList<>();
     ByteStringRange partition = ByteStringRange.create("AA", "B");
     parentPartitions.add(partition);
 
     ByteStringRange childPartition = ByteStringRange.create("A", "AB");
 
-    assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
-  public void testParentEndKeyIsBeforeChildEndKeyIsNotSuperSet() {
+  public void testParentEndKeyIsBeforeChildEndKeyDoesNotCoverKeySpace() {
     List<ByteStringRange> parentPartitions = new ArrayList<>();
     ByteStringRange partition = ByteStringRange.create("A", "B");
     parentPartitions.add(partition);
 
     ByteStringRange childPartition = ByteStringRange.create("AA", "BA");
 
-    assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
-  public void testParentIsSameAsChildIsSuperSet() {
+  public void testParentIsSameAsChildCoverKeySpace() {
     List<ByteStringRange> parentPartitions = new ArrayList<>();
     ByteStringRange partition = ByteStringRange.create("A", "B");
     parentPartitions.add(partition);
 
     ByteStringRange childPartition = ByteStringRange.create("A", "B");
 
-    assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertTrue(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
-  public void testParentIsMissingPartitionIsNotSuperSet() {
+  public void testParentIsMissingPartitionDoesNotCoverKeySpace() {
     ByteStringRange partition1 = ByteStringRange.create("A", "B");
     ByteStringRange partition2 = ByteStringRange.create("C", "Z");
     List<ByteStringRange> parentPartitions = Arrays.asList(partition1, partition2);
 
     ByteStringRange childPartition = ByteStringRange.create("A", "Z");
 
-    assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
-  public void testParentHasOverlapIsSuperSet() {
+  public void testParentHasOverlapDoesNotCoverKeySpace() {
     ByteStringRange partition1 = ByteStringRange.create("A", "C");
     ByteStringRange partition2 = ByteStringRange.create("B", "Z");
     List<ByteStringRange> parentPartitions = Arrays.asList(partition1, partition2);
 
     ByteStringRange childPartition = ByteStringRange.create("A", "Z");
 
-    assertTrue(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
-  public void testEmptyParentsIsNotSuperset() {
+  public void testEmptyParentsDoesNotCoverKeySpace() {
     List<ByteStringRange> parentPartitions = Collections.emptyList();
     ByteStringRange childPartition = ByteStringRange.create("", "");
 
-    assertFalse(ByteStringRangeHelper.isSuperset(parentPartitions, childPartition));
+    assertFalse(ByteStringRangeHelper.coverSameKeySpace(parentPartitions, childPartition));
   }
 
   @Test
@@ -181,81 +173,213 @@ public class ByteStringRangeHelperTest {
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionFromKeySpaceEmptyPartition() {
+  public void testGetMissingPartitionsEmptyPartition() {
     List<ByteStringRange> partitions = new ArrayList<>();
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
+    List<ByteStringRange> missingPartitions =
+        getMissingPartitionsFrom(
+            partitions, ByteString.copyFromUtf8("A"), ByteString.copyFromUtf8("B"));
+    assertEquals(Collections.singletonList(ByteStringRange.create("A", "B")), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionFromPartitions() {
+    List<ByteStringRange> partitions =
+        Arrays.asList(ByteStringRange.create("A", "B"), ByteStringRange.create("C", "D"));
+    List<ByteStringRange> missingPartitions =
+        getMissingPartitionsFrom(
+            partitions, ByteString.copyFromUtf8("A"), ByteString.copyFromUtf8("D"));
+    assertEquals(Collections.singletonList(ByteStringRange.create("B", "C")), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromOverlappingStart() {
+    List<ByteStringRange> partitions =
+        Arrays.asList(ByteStringRange.create("A", "C"), ByteStringRange.create("E", "F"));
+    List<ByteStringRange> missingPartitions =
+        getMissingPartitionsFrom(
+            partitions, ByteString.copyFromUtf8("B"), ByteString.copyFromUtf8("G"));
+    assertEquals(
+        Arrays.asList(ByteStringRange.create("C", "E"), ByteStringRange.create("F", "G")),
+        missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromOverlappingEnd() {
+    List<ByteStringRange> partitions =
+        Arrays.asList(ByteStringRange.create("A", "C"), ByteStringRange.create("E", "G"));
+    List<ByteStringRange> missingPartitions =
+        getMissingPartitionsFrom(
+            partitions, ByteString.copyFromUtf8("A"), ByteString.copyFromUtf8("F"));
+    assertEquals(Collections.singletonList(ByteStringRange.create("C", "E")), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromOverlappingPartitions() {
+    List<ByteStringRange> partitions =
+        Arrays.asList(ByteStringRange.create("A", "C"), ByteStringRange.create("B", "G"));
+    List<ByteStringRange> missingPartitions =
+        getMissingPartitionsFrom(
+            partitions, ByteString.copyFromUtf8("A"), ByteString.copyFromUtf8("F"));
+    assertEquals(Collections.emptyList(), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromPartitionsOutsideRange() {
+    List<ByteStringRange> partitions =
+        Arrays.asList(ByteStringRange.create("A", "B"), ByteStringRange.create("C", "D"));
+    List<ByteStringRange> missingPartitions =
+        getMissingPartitionsFrom(
+            partitions, ByteString.copyFromUtf8("G"), ByteString.copyFromUtf8("H"));
+    assertEquals(Collections.singletonList(ByteStringRange.create("G", "H")), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromPartitionsPartiallyOutsideRange() {
+    List<ByteStringRange> partitions =
+        Arrays.asList(ByteStringRange.create("A", "B"), ByteStringRange.create("D", "E"));
+    List<ByteStringRange> missingPartitions =
+        getMissingPartitionsFrom(
+            partitions, ByteString.copyFromUtf8("C"), ByteString.copyFromUtf8("H"));
     assertEquals(
-        Collections.singletonList(ByteStringRange.create("", "")), missingAndOverlappingPartitions);
+        Arrays.asList(ByteStringRange.create("C", "D"), ByteStringRange.create("E", "H")),
+        missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromKeySpaceEmptyPartition() {
+    List<ByteStringRange> partitions = new ArrayList<>();
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.singletonList(ByteStringRange.create("", "")), missingPartitions);
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionFromKeySpaceSinglePartition() {
+  public void testGetMissingPartitionsFromKeySpaceSinglePartition() {
     ByteStringRange partition1 = ByteStringRange.create("", "");
     List<ByteStringRange> partitions = Collections.singletonList(partition1);
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
-    assertEquals(Collections.emptyList(), missingAndOverlappingPartitions);
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.emptyList(), missingPartitions);
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionFromKeySpaceNoMissingPartition() {
+  public void testGetMissingPartitionsFromKeySpaceNoMissingPartition() {
     ByteStringRange partition1 = ByteStringRange.create("", "A");
     ByteStringRange partition2 = ByteStringRange.create("A", "B");
     ByteStringRange partition3 = ByteStringRange.create("B", "");
     List<ByteStringRange> partitions = Arrays.asList(partition1, partition2, partition3);
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
-    assertEquals(Collections.emptyList(), missingAndOverlappingPartitions);
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.emptyList(), missingPartitions);
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionFromKeySpaceMissingStartPartition() {
+  public void testGetMissingPartitionsFromKeySpaceMissingStartPartition() {
     ByteStringRange partition1 = ByteStringRange.create("", "A");
     ByteStringRange partition2 = ByteStringRange.create("A", "B");
     ByteStringRange partition3 = ByteStringRange.create("B", "");
     List<ByteStringRange> partitions = Arrays.asList(partition2, partition3);
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
-    assertEquals(Collections.singletonList(partition1), missingAndOverlappingPartitions);
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.singletonList(partition1), missingPartitions);
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionFromKeySpaceMissingEndPartition() {
+  public void testGetMissingPartitionsFromKeySpaceMissingEndPartition() {
     ByteStringRange partition1 = ByteStringRange.create("", "A");
     ByteStringRange partition2 = ByteStringRange.create("A", "B");
     ByteStringRange partition3 = ByteStringRange.create("B", "");
     List<ByteStringRange> partitions = Arrays.asList(partition1, partition2);
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
-    assertEquals(Collections.singletonList(partition3), missingAndOverlappingPartitions);
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.singletonList(partition3), missingPartitions);
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionFromKeySpaceMissingMiddlePartition() {
+  public void testGetMissingPartitionsFromKeySpaceMissingMiddlePartition() {
     ByteStringRange partition1 = ByteStringRange.create("", "A");
     ByteStringRange partition2 = ByteStringRange.create("A", "B");
     ByteStringRange partition3 = ByteStringRange.create("B", "");
     List<ByteStringRange> partitions = Arrays.asList(partition1, partition3);
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
-    assertEquals(Collections.singletonList(partition2), missingAndOverlappingPartitions);
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.singletonList(partition2), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromKeySpaceOverlapPartition() {
+    ByteStringRange partition1 = ByteStringRange.create("", "B");
+    ByteStringRange partition2 = ByteStringRange.create("A", "");
+    List<ByteStringRange> partitions = Arrays.asList(partition1, partition2);
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.emptyList(), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromKeySpaceOverlapAndMissingPartition() {
+    ByteStringRange partition1 = ByteStringRange.create("", "B");
+    ByteStringRange partition2 = ByteStringRange.create("C", "D");
+    ByteStringRange partition3 = ByteStringRange.create("A", "C");
+    ByteStringRange partition4 = ByteStringRange.create("E", "");
+    ByteStringRange partition5 = ByteStringRange.create("C", "E");
+    List<ByteStringRange> partitions =
+        Arrays.asList(partition1, partition2, partition3, partition4, partition5);
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.emptyList(), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromKeySpaceOverlapWithOpenEndKey() {
+    ByteStringRange fullKeySpace = ByteStringRange.create("", "");
+    ByteStringRange partialKeySpace = ByteStringRange.create("n", "");
+    List<ByteStringRange> partitions = Arrays.asList(fullKeySpace, partialKeySpace);
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.emptyList(), missingPartitions);
+  }
+
+  @Test
+  public void testGetMissingPartitionsFromEntireKeySpaceWithOverlappingPartitions() {
+    List<ByteStringRange> partitions =
+        Arrays.asList(
+            ByteStringRange.create("", "A"),
+            ByteStringRange.create("B", "C"),
+            ByteStringRange.create("A", "D"),
+            ByteStringRange.create("D", ""));
+    List<ByteStringRange> missingPartitions = getMissingPartitionsFromEntireKeySpace(partitions);
+    assertEquals(Collections.emptyList(), missingPartitions);
+  }
+
+  @Test
+  public void testGetOverlappingPartitionsFromKeySpaceEmptyPartition() {
+    List<ByteStringRange> partitions = new ArrayList<>();
+    List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
+    assertEquals(Collections.emptyList(), overlappingPartitions);
+  }
+
+  @Test
+  public void testGetOverlappingPartitionsSinglePartition() {
+    ByteStringRange partition1 = ByteStringRange.create("", "");
+    List<ByteStringRange> partitions = Collections.singletonList(partition1);
+    List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
+    assertEquals(Collections.emptyList(), overlappingPartitions);
+  }
+
+  @Test
+  public void testGetOverlappingPartitionsNoMissingPartition() {
+    ByteStringRange partition1 = ByteStringRange.create("", "A");
+    ByteStringRange partition2 = ByteStringRange.create("A", "B");
+    ByteStringRange partition3 = ByteStringRange.create("B", "");
+    List<ByteStringRange> partitions = Arrays.asList(partition1, partition2, partition3);
+    List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
+    assertEquals(Collections.emptyList(), overlappingPartitions);
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionFromKeySpaceOverlapPartition() {
+  public void testGetOverlappingPartitionsOverlapPartition() {
     ByteStringRange partition1 = ByteStringRange.create("", "B");
     ByteStringRange partition2 = ByteStringRange.create("A", "");
     List<ByteStringRange> partitions = Arrays.asList(partition1, partition2);
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
+    List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
     assertEquals(
-        Collections.singletonList(ByteStringRange.create("B", "A")),
-        missingAndOverlappingPartitions);
+        Collections.singletonList(ByteStringRange.create("A", "B")), overlappingPartitions);
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionFromKeySpaceOverlapAndMissingPartition() {
+  public void testGetOverlappingPartitionsOverlapAndMissingPartition() {
     ByteStringRange partition1 = ByteStringRange.create("", "B");
     ByteStringRange partition2 = ByteStringRange.create("C", "D");
     ByteStringRange partition3 = ByteStringRange.create("A", "C");
@@ -263,25 +387,19 @@ public class ByteStringRangeHelperTest {
     ByteStringRange partition5 = ByteStringRange.create("C", "E");
     List<ByteStringRange> partitions =
         Arrays.asList(partition1, partition2, partition3, partition4, partition5);
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
+    List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
     assertEquals(
-        Arrays.asList(ByteStringRange.create("B", "A"), ByteStringRange.create("D", "C")),
-        missingAndOverlappingPartitions);
+        Arrays.asList(ByteStringRange.create("A", "B"), ByteStringRange.create("C", "D")),
+        overlappingPartitions);
   }
 
   @Test
-  public void testGetMissingAndOverlappingPartitionsFromKeySpaceOverlapWithOpenEndKey() {
+  public void testGetOverlappingPartitions() {
     ByteStringRange fullKeySpace = ByteStringRange.create("", "");
     ByteStringRange partialKeySpace = ByteStringRange.create("n", "");
     List<ByteStringRange> partitions = Arrays.asList(fullKeySpace, partialKeySpace);
-    // TODO come up with a better way to differentiate missing with start key "" and overlapping
-    // with end key ""
-    ByteStringRange overlappingPartition = ByteStringRange.create("", "n");
-    List<ByteStringRange> expectedOverlapping = Collections.singletonList(overlappingPartition);
-    List<ByteStringRange> missingAndOverlappingPartitions =
-        getMissingAndOverlappingPartitionsFromKeySpace(partitions);
-    assertEquals(expectedOverlapping, missingAndOverlappingPartitions);
+    List<ByteStringRange> overlappingPartitions = getOverlappingPartitions(partitions);
+    assertEquals(Collections.singletonList(ByteStringRange.create("n", "")), overlappingPartitions);
   }
 
   @Test
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverterTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverterTest.java
index 5c8b916ffc4..08b85980263 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverterTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/TimestampConverterTest.java
@@ -17,6 +17,7 @@
  */
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams;
 
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.microsecondToInstant;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.toJodaTime;
 import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConverter.toThreetenInstant;
 import static org.junit.Assert.assertEquals;
@@ -41,4 +42,9 @@ public class TimestampConverterTest {
     assertEquals(1, TimestampConverter.toSeconds(org.joda.time.Instant.ofEpochSecond(1)));
     assertEquals(1000, TimestampConverter.toSeconds(org.joda.time.Instant.ofEpochSecond(1000)));
   }
+
+  @Test
+  public void testMicrosecondToJodaInstant() {
+    assertEquals(org.joda.time.Instant.ofEpochMilli(1_234L), microsecondToInstant(1_234_567L));
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java
index 7724dad1209..6fe272009cc 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ChangeStreamActionTest.java
@@ -123,8 +123,8 @@ public class ChangeStreamActionTest {
   public void testChangeStreamMutationUser() {
     ByteStringRange partition = ByteStringRange.create("", "");
     when(partitionRecord.getPartition()).thenReturn(partition);
-    final Instant commitTimestamp = Instant.ofEpochSecond(1000);
-    final Instant lowWatermark = Instant.ofEpochSecond(500);
+    final Instant commitTimestamp = Instant.ofEpochMilli(1_000L);
+    final Instant lowWatermark = Instant.ofEpochMilli(500L);
     ChangeStreamContinuationToken changeStreamContinuationToken =
         ChangeStreamContinuationToken.create(ByteStringRange.create("", ""), "1234");
     ChangeStreamMutation changeStreamMutation = Mockito.mock(ChangeStreamMutation.class);
@@ -155,8 +155,8 @@ public class ChangeStreamActionTest {
   public void testChangeStreamMutationGc() {
     ByteStringRange partition = ByteStringRange.create("", "");
     when(partitionRecord.getPartition()).thenReturn(partition);
-    final Instant commitTimestamp = Instant.ofEpochSecond(1000);
-    final Instant lowWatermark = Instant.ofEpochSecond(500);
+    final Instant commitTimestamp = Instant.ofEpochMilli(1_000L);
+    final Instant lowWatermark = Instant.ofEpochMilli(500L);
     ChangeStreamContinuationToken changeStreamContinuationToken =
         ChangeStreamContinuationToken.create(ByteStringRange.create("", ""), "1234");
     ChangeStreamMutation changeStreamMutation = Mockito.mock(ChangeStreamMutation.class);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java
index a3d741dcdd8..9d8aee40ed3 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/DetectNewPartitionsActionTest.java
@@ -20,6 +20,8 @@ package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.never;
@@ -33,9 +35,11 @@ import com.google.cloud.bigtable.data.v2.BigtableDataClient;
 import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
 import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
 import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
 import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -43,12 +47,11 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
 import org.apache.beam.sdk.io.range.OffsetRange;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.sdk.transforms.DoFn.BundleFinalizer;
-import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
 import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
 import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
 import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators;
@@ -72,17 +75,16 @@ public class DetectNewPartitionsActionTest {
 
   @Mock private ChangeStreamMetrics metrics;
   @Mock private GenerateInitialPartitionsAction generateInitialPartitionsAction;
+  @Mock private ResumeFromPreviousPipelineAction resumeFromPreviousPipelineAction;
 
   private DetectNewPartitionsAction action;
 
   @Mock private RestrictionTracker<OffsetRange, Long> tracker;
-  @Mock private OutputReceiver<PartitionRecord> receiver;
-  @Mock private BundleFinalizer bundleFinalizer;
+  @Mock private DoFn.OutputReceiver<PartitionRecord> receiver;
 
   private MetadataTableDao metadataTableDao;
   private ManualWatermarkEstimator<Instant> watermarkEstimator;
   private Instant startTime;
-  private Instant partitionTime;
   private Instant endTime;
   private static BigtableDataClient dataClient;
   private static BigtableTableAdminClient adminClient;
@@ -120,125 +122,407 @@ public class DetectNewPartitionsActionTest {
 
     startTime = Instant.now();
     endTime = startTime.plus(Duration.standardSeconds(10));
-    partitionTime = startTime.plus(Duration.standardSeconds(10));
+    ProcessNewPartitionsAction processNewPartitionsAction =
+        new ProcessNewPartitionsAction(metrics, metadataTableDao, endTime);
     action =
         new DetectNewPartitionsAction(
-            metrics, metadataTableDao, endTime, generateInitialPartitionsAction);
+            metrics,
+            metadataTableDao,
+            endTime,
+            generateInitialPartitionsAction,
+            resumeFromPreviousPipelineAction,
+            processNewPartitionsAction);
     watermarkEstimator = new WatermarkEstimators.Manual(startTime);
   }
 
   @Test
-  public void testInitialPartitions() throws Exception {
-    OffsetRange offsetRange = new OffsetRange(0, Long.MAX_VALUE);
+  public void testUpdateWatermarkAfterCheckpoint() throws Exception {
+    Instant watermark = endTime;
+    OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
+    when(tracker.currentRestriction()).thenReturn(offsetRange);
+    when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+
+    // Watermark estimator is not updated before a checkpoint.
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+    // Update the watermark in the metadata table. This run terminates because watermark == endTime.
+    metadataTableDao.updateDetectNewPartitionWatermark(watermark);
+    // Watermark estimator will be updated with the watermark from the metadata table.
+    assertEquals(
+        DoFn.ProcessContinuation.stop(),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(watermark, watermarkEstimator.currentWatermark());
+    assertEquals(watermark, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+  }
+
+  @Test
+  public void testUpdateWatermarkOnEvenCountAfter10Seconds() throws Exception {
+    // We update watermark every 2 iterations only if it's been more than 10s since the last update.
+    OffsetRange offsetRange = new OffsetRange(2, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
+    when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
     assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+    // Manually set the watermark of DNP to start time with a timestamp of 10s prior.
+    RowMutation rowMutation =
+        RowMutation.create(
+                MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME,
+                metadataTableDao
+                    .getChangeStreamNamePrefix()
+                    .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX))
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK,
+                MetadataTableAdminDao.QUALIFIER_DEFAULT,
+                Instant.now().minus(Duration.standardSeconds(10)).getMillis() * 1000L,
+                startTime.getMillis());
+    dataClient.mutateRow(rowMutation);
+
+    // Create a partition covering the entire keyspace with watermark after endTime.
+    ByteStringRange partition1 = ByteStringRange.create("", "");
+    Instant watermark1 = endTime.plus(Duration.millis(100));
+    PartitionRecord partitionRecord1 =
+        new PartitionRecord(
+            partition1,
+            watermark1,
+            UniqueIdGenerator.getNextId(),
+            watermark1,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord1);
 
-    when(generateInitialPartitionsAction.run(receiver, tracker, watermarkEstimator, startTime))
-        .thenReturn(ProcessContinuation.resume());
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertEquals(watermark1, metadataTableDao.readDetectNewPartitionsState().getWatermark());
 
+    // On the 2nd run, watermark estimator is updated which is beyond endTime and terminates.
     assertEquals(
-        ProcessContinuation.resume(),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+        DoFn.ProcessContinuation.stop(),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    verify(tracker, times(1)).tryClaim(offsetRange.getTo());
+    assertEquals(watermark1, watermarkEstimator.currentWatermark());
   }
 
-  // Every 10 tryClaim, DNP updates the watermark based on the watermark of all the RCSP.
+  @Test
+  public void testDoNotUpdateWatermarkLessThan10s() throws Exception {
+    // We update watermark every 2 iterations only if it's been more than 10s since the last update.
+    OffsetRange offsetRange = new OffsetRange(2, Long.MAX_VALUE);
+    when(tracker.currentRestriction()).thenReturn(offsetRange);
+    when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+    // Manually set the watermark of DNP to start time with a timestamp of 1s prior.
+    RowMutation rowMutation =
+        RowMutation.create(
+                MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME,
+                metadataTableDao
+                    .getChangeStreamNamePrefix()
+                    .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX))
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK,
+                MetadataTableAdminDao.QUALIFIER_DEFAULT,
+                Instant.now().minus(Duration.standardSeconds(1)).getMillis() * 1000L,
+                startTime.getMillis());
+    dataClient.mutateRow(rowMutation);
+
+    // Create a partition covering the entire keyspace with watermark after endTime.
+    ByteStringRange partition1 = ByteStringRange.create("", "");
+    Instant watermark1 = endTime.plus(Duration.millis(100));
+    PartitionRecord partitionRecord1 =
+        new PartitionRecord(
+            partition1,
+            watermark1,
+            UniqueIdGenerator.getNextId(),
+            watermark1,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord1);
+
+    // Watermark doesn't get updated because the last time watermark was updated was less than 10s.
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertEquals(startTime, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+
+    // On the 2nd run, watermark estimator is still the same since watermark isn't updated.
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+  }
+
+  @Test
+  public void testDoNotUpdateWatermarkOnOddCount() throws Exception {
+    // We only update watermark on even count.
+    OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
+    when(tracker.currentRestriction()).thenReturn(offsetRange);
+    when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+    // Manually set the watermark of DNP to start time with a timestamp of 10s prior.
+    RowMutation rowMutation =
+        RowMutation.create(
+                MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME,
+                metadataTableDao
+                    .getChangeStreamNamePrefix()
+                    .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX))
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK,
+                MetadataTableAdminDao.QUALIFIER_DEFAULT,
+                Instant.now().minus(Duration.standardSeconds(10)).getMillis() * 1000L,
+                startTime.getMillis());
+    dataClient.mutateRow(rowMutation);
+
+    // Create a partition covering the entire keyspace with watermark after endTime.
+    ByteStringRange partition1 = ByteStringRange.create("", "");
+    Instant watermark1 = endTime.plus(Duration.millis(100));
+    PartitionRecord partitionRecord1 =
+        new PartitionRecord(
+            partition1,
+            watermark1,
+            UniqueIdGenerator.getNextId(),
+            watermark1,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord1);
+
+    // It's been more than 10s since last update but the count is odd.
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertEquals(startTime, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+
+    // On the 2nd run, watermark estimator is still the same since watermark isn't updated.
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+  }
+
+  // Every 2 tryClaim, DNP updates the watermark based on the watermark of all the RCSP.
   @Test
   public void testAdvanceWatermarkWithAllPartitions() throws Exception {
-    // We advance watermark on every 10 restriction tracker advancement
+    // We advance watermark on every 2 restriction tracker advancement
     OffsetRange offsetRange = new OffsetRange(10, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
+    when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+    when(tracker.tryClaim(offsetRange.getTo())).thenReturn(true);
 
     assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertNull(metadataTableDao.readDetectNewPartitionsState());
 
     // Write 2 partitions to the table that covers entire keyspace.
     ByteStringRange partition1 = ByteStringRange.create("", "b");
-    Instant watermark1 = partitionTime.plus(Duration.millis(100));
-    metadataTableDao.updateWatermark(partition1, watermark1, null);
+    Instant watermark1 = endTime.plus(Duration.millis(100));
+    PartitionRecord partitionRecord1 =
+        new PartitionRecord(
+            partition1,
+            watermark1,
+            UniqueIdGenerator.getNextId(),
+            watermark1,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord1);
     ByteStringRange partition2 = ByteStringRange.create("b", "");
     Instant watermark2 = endTime.plus(Duration.millis(1));
-    metadataTableDao.updateWatermark(partition2, watermark2, null);
+    PartitionRecord partitionRecord2 =
+        new PartitionRecord(
+            partition2,
+            watermark2,
+            UniqueIdGenerator.getNextId(),
+            watermark2,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord2);
+
+    // Updating watermark does not affect the watermark estimator of this run.
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    verify(tracker, times(1)).tryClaim(offsetRange.getFrom());
+
+    // Because the 2 partitions cover the entire keyspace, the watermark should have advanced.
+    // Also note the watermark is watermark2 which is the lowest of the 2 watermarks. Watermark
+    // estimator isn't updated because we update the watermark estimator on the start of the run. We
+    // do update the metadata table with the new watermark value.
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertEquals(watermark2, metadataTableDao.readDetectNewPartitionsState().getWatermark());
 
+    // On the 2nd run, watermark estimator is updated which is beyond endTime and terminates.
     assertEquals(
         DoFn.ProcessContinuation.stop(),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
-    // TryClaim should be called with the end value because the watermark should be updated to
-    // beyond endTime causing the pipeline to terminate.
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
     verify(tracker, times(1)).tryClaim(offsetRange.getTo());
-
-    // Because the 2 partitions cover the entire keyspace, the watermark should have advanced.
-    // Also note the watermark is watermark2 which is the lowest of the 2 watermarks.
     assertEquals(watermark2, watermarkEstimator.currentWatermark());
   }
 
-  // Every 10 tryClaim, DNP only updates its watermark if all the RCSP currently streamed covers the
+  // Every 2 tryClaim, DNP only updates its watermark if all the RCSP currently streamed covers the
   // entire key space. If there's any missing, they are in the process of split or merge. If the
   // watermark is updated with missing partitions, the watermark might be further ahead than it
   // actually is.
   @Test
   public void testAdvanceWatermarkWithMissingPartitions() throws Exception {
-    // We advance watermark on every 10 restriction tracker advancement
-    OffsetRange offsetRange = new OffsetRange(10, Long.MAX_VALUE);
+    // We advance watermark on every 2 restriction tracker advancement
+    OffsetRange offsetRange = new OffsetRange(2, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
-    // We return false here, so we do need to test any other functionality other than updating the
-    // watermark.
-    when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(false);
+    when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
 
     assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertNull(metadataTableDao.readDetectNewPartitionsState());
 
     // Write 2 partitions to the table that DO NOT cover the entire keyspace.
     ByteStringRange partition1 = ByteStringRange.create("", "b");
-    Instant watermark1 = partitionTime.plus(Duration.millis(100));
+    Instant watermark1 = endTime.plus(Duration.millis(100));
     metadataTableDao.updateWatermark(partition1, watermark1, null);
     ByteStringRange partition2 = ByteStringRange.create("b", "c");
     Instant watermark2 = endTime.plus(Duration.millis(1));
     metadataTableDao.updateWatermark(partition2, watermark2, null);
 
     assertEquals(
-        DoFn.ProcessContinuation.stop(),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
-    // Ensure that we did indeed try to claim the start of offset range because that means watermark
-    // was not updated beyond endTime.
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
     verify(tracker, times(1)).tryClaim(offsetRange.getFrom());
 
     // Because the 2 partitions DO NOT cover the entire keyspace, watermark stays at startTime.
     assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertNull(metadataTableDao.readDetectNewPartitionsState());
+  }
+
+  @Test
+  public void testAdvanceWatermarkWithNewPartitions() throws Exception {
+    // We advance watermark on every 2 restriction tracker advancement
+    OffsetRange offsetRange = new OffsetRange(2, Long.MAX_VALUE);
+    when(tracker.currentRestriction()).thenReturn(offsetRange);
+    when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
+    when(tracker.tryClaim(offsetRange.getTo())).thenReturn(true);
+
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertNull(metadataTableDao.readDetectNewPartitionsState());
+
+    // Write 2 partitions to the table that DO NOT cover the entire keyspace.
+    ByteStringRange partition1 = ByteStringRange.create("", "b");
+    Instant watermark1 = endTime.plus(Duration.millis(100));
+    PartitionRecord partitionRecord1 =
+        new PartitionRecord(
+            partition1,
+            watermark1,
+            UniqueIdGenerator.getNextId(),
+            watermark1,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord1);
+    ByteStringRange partition2 = ByteStringRange.create("b", "c");
+    Instant watermark2 = endTime.plus(Duration.millis(1));
+    PartitionRecord partitionRecord2 =
+        new PartitionRecord(
+            partition2,
+            watermark2,
+            UniqueIdGenerator.getNextId(),
+            watermark2,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord2);
+
+    ByteStringRange partition3 = ByteStringRange.create("c", "");
+    ChangeStreamContinuationToken token = ChangeStreamContinuationToken.create(partition3, "token");
+    NewPartition newPartition =
+        new NewPartition(partition3, Collections.singletonList(token), watermark2);
+    metadataTableDao.writeNewPartition(newPartition);
+
+    // Updating watermark does not affect the result of this run.
+    assertEquals(
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    verify(tracker, times(1)).tryClaim(offsetRange.getFrom());
+
+    // Because the StreamPartition and NewPartition cover the entire keyspace, the watermark should
+    // have advanced. Also note the watermark is watermark2 which is the lowest watermark. Watermark
+    // estimator isn't updated because we update the watermark estimator on the start of the run. We
+    // do update the metadata table with the new watermark value.
+    assertEquals(startTime, watermarkEstimator.currentWatermark());
+    assertEquals(watermark2, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+
+    // On the 2nd run, watermark estimator is updated which is beyond endTime and terminates.
+    assertEquals(
+        DoFn.ProcessContinuation.stop(),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    verify(tracker, times(1)).tryClaim(offsetRange.getTo());
+    assertEquals(watermark2, watermarkEstimator.currentWatermark());
   }
 
   @Test
   public void testProcessSplitNewPartitions() throws Exception {
-    // Avoid 0 and multiples of 10 so that we can specifically test just reading new partitions.
+    // Avoid 0 and multiples of 2 so that we can specifically test just reading new partitions.
     OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
     when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
 
-    // ["", "") splits into ["", "a"), ["a","c"), ["c", "")
-    ByteStringRange parentPartition = ByteStringRange.create("", "");
+    // ["", "") splits into ["", "a"), ["a","k"), ["k", "")
     Instant watermark = Instant.now();
-    ByteStringRange child1 = ByteStringRange.create("", "a");
-    metadataTableDao.writeNewPartition(
-        child1,
+    ByteStringRange childEmptyA = ByteStringRange.create("", "a");
+    ChangeStreamContinuationToken tokenEmptyA =
         ChangeStreamContinuationToken.create(
-            child1, ByteStringRange.serializeToByteString(child1).toStringUtf8()),
-        parentPartition,
-        watermark);
-    ByteStringRange child2 = ByteStringRange.create("k", "");
-    metadataTableDao.writeNewPartition(
-        child2,
+            childEmptyA, ByteStringRange.serializeToByteString(childEmptyA).toStringUtf8());
+    NewPartition newPartitionEmptyA =
+        new NewPartition(childEmptyA, Collections.singletonList(tokenEmptyA), watermark);
+    metadataTableDao.writeNewPartition(newPartitionEmptyA);
+    ByteStringRange childKEmpty = ByteStringRange.create("k", "");
+    ChangeStreamContinuationToken tokenKEmpty =
         ChangeStreamContinuationToken.create(
-            child2, ByteStringRange.serializeToByteString(child2).toStringUtf8()),
-        parentPartition,
-        watermark);
-    ByteStringRange child3 = ByteStringRange.create("a", "k");
-    metadataTableDao.writeNewPartition(
-        child3,
+            childKEmpty, ByteStringRange.serializeToByteString(childKEmpty).toStringUtf8());
+    NewPartition newPartitionKEmpty =
+        new NewPartition(childKEmpty, Collections.singletonList(tokenKEmpty), watermark);
+    metadataTableDao.writeNewPartition(newPartitionKEmpty);
+    ByteStringRange childAK = ByteStringRange.create("a", "k");
+    ChangeStreamContinuationToken tokenAK =
         ChangeStreamContinuationToken.create(
-            child3, ByteStringRange.serializeToByteString(child3).toStringUtf8()),
-        parentPartition,
-        watermark);
+            childAK, ByteStringRange.serializeToByteString(childAK).toStringUtf8());
+    NewPartition newPartitionAK =
+        new NewPartition(childAK, Collections.singletonList(tokenAK), watermark);
+    metadataTableDao.writeNewPartition(newPartitionAK);
 
     assertEquals(
-        ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
 
     // Check what parameters were passed to OutputReceiver to verify the new partitions created by
     // DNP are correct.
@@ -248,7 +532,7 @@ public class DetectNewPartitionsActionTest {
         partitionRecordArgumentCaptor.getAllValues().stream()
             .map(PartitionRecord::getPartition)
             .collect(Collectors.toList());
-    List<ByteStringRange> actualPartitions = Arrays.asList(child1, child2, child3);
+    List<ByteStringRange> actualPartitions = Arrays.asList(childEmptyA, childKEmpty, childAK);
     assertThat(expectedPartitions, Matchers.containsInAnyOrder(actualPartitions.toArray()));
     for (PartitionRecord value : partitionRecordArgumentCaptor.getAllValues()) {
       assertEquals(value.getParentLowWatermark(), watermark);
@@ -264,143 +548,181 @@ public class DetectNewPartitionsActionTest {
                   value.getChangeStreamContinuationTokens().get(0).getPartition())
               .toStringUtf8());
     }
+    assertTrue(metadataTableDao.readNewPartitions().isEmpty());
   }
 
   @Test
   public void testProcessMergeNewPartitions() throws Exception {
-    // Avoid 0 and multiples of 10 so that we can specifically test just reading new partitions.
+    // Avoid 0 and multiples of 2 so that we can specifically test just reading new partitions.
     OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
     when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
 
     // ["a, "b") and ["b", "c") merge into ["a", "c")
-    ByteStringRange parentPartition1 = ByteStringRange.create("a", "b");
-    Instant watermark1 = startTime;
-    ByteStringRange parentPartition2 = ByteStringRange.create("b", "c");
-    Instant watermark2 = startTime.plus(Duration.millis(10));
-
-    ByteStringRange childPartition = ByteStringRange.create("a", "c");
-    ChangeStreamContinuationToken token1 =
-        ChangeStreamContinuationToken.create(childPartition, "token1");
-    ChangeStreamContinuationToken token2 =
-        ChangeStreamContinuationToken.create(childPartition, "token2");
+    ByteStringRange childPartitionAC = ByteStringRange.create("a", "c");
+
+    ByteStringRange parentPartitionAB = ByteStringRange.create("a", "b");
+    Instant watermarkAB = startTime;
+    ChangeStreamContinuationToken tokenAB =
+        ChangeStreamContinuationToken.create(parentPartitionAB, "ab");
+    NewPartition newPartitionACFromAB =
+        new NewPartition(childPartitionAC, Collections.singletonList(tokenAB), watermarkAB);
+    ByteStringRange parentPartitionBC = ByteStringRange.create("b", "c");
+    Instant watermarkBC = startTime.plus(Duration.millis(10));
+    ChangeStreamContinuationToken tokenBC =
+        ChangeStreamContinuationToken.create(parentPartitionBC, "bc");
+    NewPartition newPartitionACFromBC =
+        new NewPartition(childPartitionAC, Collections.singletonList(tokenBC), watermarkBC);
     // Write a new partition for every parent partition that merges into the child.
-    metadataTableDao.writeNewPartition(childPartition, token1, parentPartition1, watermark1);
-    metadataTableDao.writeNewPartition(childPartition, token2, parentPartition2, watermark2);
+    metadataTableDao.writeNewPartition(newPartitionACFromAB);
+    metadataTableDao.writeNewPartition(newPartitionACFromBC);
 
     assertEquals(
-        ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
     // The partition is outputted with watermark1 because that is the lowest of the 2 forming the
     // parent low watermark.
     verify(receiver, times(1))
         .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
 
-    assertEquals(childPartition, partitionRecordArgumentCaptor.getValue().getPartition());
-    assertEquals(watermark1, partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
+    assertEquals(childPartitionAC, partitionRecordArgumentCaptor.getValue().getPartition());
+    assertEquals(watermarkAB, partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
     assertEquals(endTime, partitionRecordArgumentCaptor.getValue().getEndTime());
     assertThat(
         partitionRecordArgumentCaptor.getValue().getChangeStreamContinuationTokens(),
-        Matchers.containsInAnyOrder(token1, token2));
+        Matchers.containsInAnyOrder(tokenAB, tokenBC));
+    assertTrue(metadataTableDao.readNewPartitions().isEmpty());
   }
 
   // Test merging partition with a parent partition that hasn't stopped yet.
   @Test
   public void testProcessMergeNewPartitionsMissingParent() throws Exception {
-    // Avoid 0 and multiples of 10 so that we can specifically test just reading new partitions.
+    // Avoid 0 and multiples of 2 so that we can specifically test just reading new partitions.
     OffsetRange offsetRange = new OffsetRange(1, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
     when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
 
     // ["a, "b") and ["b", "c") merge into ["a", "c") but ["b", "c") is still processing.
-    ByteStringRange parentPartition1 = ByteStringRange.create("a", "b");
-    Instant watermark1 = startTime;
+    ByteStringRange parentPartitionAB = ByteStringRange.create("a", "b");
+    Instant watermarkAB = startTime;
+    ChangeStreamContinuationToken tokenAB =
+        ChangeStreamContinuationToken.create(parentPartitionAB, "ab");
+
+    ByteStringRange childPartitionAC = ByteStringRange.create("a", "c");
+
+    NewPartition newPartitionACFromAB =
+        new NewPartition(childPartitionAC, Collections.singletonList(tokenAB), watermarkAB);
 
-    ByteStringRange childPartition = ByteStringRange.create("a", "c");
-    ChangeStreamContinuationToken token1 =
-        ChangeStreamContinuationToken.create(childPartition, "token1");
     // Write a new partition for every parent partition that merges into the child.
-    metadataTableDao.writeNewPartition(childPartition, token1, parentPartition1, watermark1);
+    metadataTableDao.writeNewPartition(newPartitionACFromAB);
 
     assertEquals(
-        ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
-    // The partition is outputted with watermark1 because that is the lowest of the 2 forming the
-    // parent low watermark.
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
+    // No partitions are outputted because it's missing a parent still.
     verify(receiver, never()).outputWithTimestamp(any(), any());
 
+    assertEquals(1, metadataTableDao.readNewPartitions().size());
+
     // On next iteration, ["b", "c") received CloseStream and writes to the metadata table.
-    ByteStringRange parentPartition2 = ByteStringRange.create("b", "c");
-    Instant watermark2 = startTime.plus(Duration.millis(10));
-    ChangeStreamContinuationToken token2 =
-        ChangeStreamContinuationToken.create(childPartition, "token2");
-    metadataTableDao.writeNewPartition(childPartition, token2, parentPartition2, watermark2);
+    ByteStringRange parentPartitionBC = ByteStringRange.create("b", "c");
+    Instant watermarkBC = startTime.plus(Duration.millis(10));
+    ChangeStreamContinuationToken tokenBC =
+        ChangeStreamContinuationToken.create(parentPartitionBC, "bc");
+    NewPartition newPartitionACFromBC =
+        new NewPartition(childPartitionAC, Collections.singletonList(tokenBC), watermarkBC);
+    metadataTableDao.writeNewPartition(newPartitionACFromBC);
 
     assertEquals(
-        ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
     // The partition is outputted with watermark1 because that is the lowest of the 2 forming the
     // parent low watermark.
     verify(receiver, times(1))
         .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
 
-    assertEquals(childPartition, partitionRecordArgumentCaptor.getValue().getPartition());
-    assertEquals(watermark1, partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
+    assertEquals(childPartitionAC, partitionRecordArgumentCaptor.getValue().getPartition());
+    assertEquals(watermarkAB, partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
     assertEquals(endTime, partitionRecordArgumentCaptor.getValue().getEndTime());
     assertThat(
         partitionRecordArgumentCaptor.getValue().getChangeStreamContinuationTokens(),
-        Matchers.containsInAnyOrder(token1, token2));
+        Matchers.containsInAnyOrder(tokenAB, tokenBC));
+    assertTrue(metadataTableDao.readNewPartitions().isEmpty());
   }
 
   @Test
   public void testMissingPartitionReconciled() throws Exception {
-    // We advance watermark on every 10 restriction tracker advancement
-    OffsetRange offsetRange = new OffsetRange(10, Long.MAX_VALUE);
+    // We only start reconciling after 50.
+    // We advance watermark on every 2 restriction tracker advancement
+    OffsetRange offsetRange = new OffsetRange(52, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
     when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
 
     // Write 2 partitions to the table, missing [a, b)
-    ByteStringRange partition1 = ByteStringRange.create("", "a");
-    Instant watermark1 = partitionTime.plus(Duration.millis(100));
-    metadataTableDao.updateWatermark(partition1, watermark1, null);
-    ByteStringRange partition2 = ByteStringRange.create("b", "");
-    Instant watermark2 = endTime.plus(Duration.millis(1));
-    metadataTableDao.updateWatermark(partition2, watermark2, null);
-
-    HashMap<ByteStringRange, Long> missingPartitionDurations = new HashMap<>();
+    ByteStringRange partitionEmptyA = ByteStringRange.create("", "a");
+    Instant watermarkEmptyA = endTime.plus(Duration.millis(100));
+    PartitionRecord partitionRecordEmptyA =
+        new PartitionRecord(
+            partitionEmptyA,
+            watermarkEmptyA,
+            UniqueIdGenerator.getNextId(),
+            watermarkEmptyA,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecordEmptyA);
+    ByteStringRange partitionBEmpty = ByteStringRange.create("b", "");
+    Instant watermarkBEmpty = endTime.plus(Duration.millis(1));
+    PartitionRecord partitionRecordBEmpty =
+        new PartitionRecord(
+            partitionBEmpty,
+            watermarkBEmpty,
+            UniqueIdGenerator.getNextId(),
+            watermarkBEmpty,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecordBEmpty);
+
+    HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
     ByteStringRange partitionAB = ByteStringRange.create("a", "b");
     // Partition missing for 5 minutes less 1 seconds.
-    missingPartitionDurations.put(partitionAB, Instant.now().getMillis() - (5 * 60 - 1) * 1000L);
+    missingPartitionDurations.put(
+        partitionAB, Instant.now().minus(Duration.standardSeconds(10 * 60 - 1)));
     metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
 
     // No new partitions and missing partition has not been missing for long enough.
     assertEquals(
-        ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
     verify(receiver, never()).outputWithTimestamp(any(), any());
 
     // Sleep for 1 second, enough that the missing partition needs to be reconciled.
-    Thread.sleep(1000);
+    Thread.sleep(1001);
 
-    // We advance the restriction tracker by 1. Because it is not a multiple of 10, we don't
+    // We advance the restriction tracker by 1. Because it is not a multiple of 2, we don't
     // evaluate missing partitions, which means we don't perform reconciliation.
-    offsetRange = new OffsetRange(11, Long.MAX_VALUE);
+    offsetRange = new OffsetRange(53, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
     when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
 
     assertEquals(
-        ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
     verify(receiver, never()).outputWithTimestamp(any(), any());
 
-    // Multiple of 10, reconciliation should happen.
-    offsetRange = new OffsetRange(20, Long.MAX_VALUE);
+    // Multiple of 2, reconciliation should happen.
+    offsetRange = new OffsetRange(54, Long.MAX_VALUE);
     when(tracker.currentRestriction()).thenReturn(offsetRange);
     when(tracker.tryClaim(offsetRange.getFrom())).thenReturn(true);
 
     assertEquals(
-        ProcessContinuation.resume().withResumeDelay(Duration.standardSeconds(1)),
-        action.run(tracker, receiver, watermarkEstimator, bundleFinalizer, startTime));
+        DoFn.ProcessContinuation.resume().withResumeDelay(Duration.millis(100)),
+        action.run(
+            tracker, receiver, watermarkEstimator, new InitialPipelineState(startTime, false)));
     verify(receiver, times(1))
         .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
     assertEquals(partitionAB, partitionRecordArgumentCaptor.getValue().getPartition());
@@ -409,8 +731,8 @@ public class DetectNewPartitionsActionTest {
         partitionRecordArgumentCaptor.getValue().getParentLowWatermark());
     assertEquals(endTime, partitionRecordArgumentCaptor.getValue().getEndTime());
     assertNotNull(partitionRecordArgumentCaptor.getValue().getStartTime());
-    assertEquals(
-        watermarkEstimator.currentWatermark(),
-        partitionRecordArgumentCaptor.getValue().getStartTime());
+    // The startTime should be the startTime of the pipeline because it's less than 1 hour before
+    // the low watermark.
+    assertEquals(startTime, partitionRecordArgumentCaptor.getValue().getStartTime());
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java
index dcaadd568f8..102200ab9a1 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/GenerateInitialPartitionsActionTest.java
@@ -25,7 +25,7 @@ import static org.mockito.Mockito.when;
 
 import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
 import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
-import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import com.google.cloud.bigtable.data.v2.models.Range;
 import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
 import java.io.IOException;
 import java.util.Arrays;
@@ -35,12 +35,7 @@ import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
-import org.apache.beam.sdk.io.range.OffsetRange;
 import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
-import org.apache.beam.sdk.transforms.DoFn.ProcessContinuation;
-import org.apache.beam.sdk.transforms.splittabledofn.ManualWatermarkEstimator;
-import org.apache.beam.sdk.transforms.splittabledofn.RestrictionTracker;
-import org.apache.beam.sdk.transforms.splittabledofn.WatermarkEstimators;
 import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
@@ -58,12 +53,10 @@ public class GenerateInitialPartitionsActionTest {
   @ClassRule
   public static final BigtableEmulatorRule BIGTABLE_EMULATOR_RULE = BigtableEmulatorRule.create();
 
-  @Mock private RestrictionTracker<OffsetRange, Long> tracker;
   @Mock private ChangeStreamDao changeStreamDao;
   @Mock private ChangeStreamMetrics metrics;
   @Mock private OutputReceiver<PartitionRecord> receiver;
 
-  private ManualWatermarkEstimator<Instant> watermarkEstimator;
   private Instant startTime;
   private Instant endTime;
   private static BigtableTableAdminClient adminClient;
@@ -89,23 +82,18 @@ public class GenerateInitialPartitionsActionTest {
     metadataTableAdminDao.createMetadataTable();
     startTime = Instant.now();
     endTime = startTime.plus(Duration.standardSeconds(10));
-    watermarkEstimator = new WatermarkEstimators.Manual(startTime);
   }
 
   @Test
   public void testGenerateInitialPartitionsFromStartTime() {
-    when(tracker.tryClaim(0L)).thenReturn(true);
-
-    ByteStringRange partition1 = ByteStringRange.create("", "b");
-    ByteStringRange partition2 = ByteStringRange.create("b", "");
-    List<ByteStringRange> partitionRecordList = Arrays.asList(partition1, partition2);
+    Range.ByteStringRange partition1 = Range.ByteStringRange.create("", "b");
+    Range.ByteStringRange partition2 = Range.ByteStringRange.create("b", "");
+    List<Range.ByteStringRange> partitionRecordList = Arrays.asList(partition1, partition2);
     when(changeStreamDao.generateInitialChangeStreamPartitions()).thenReturn(partitionRecordList);
 
     GenerateInitialPartitionsAction generateInitialPartitionsAction =
         new GenerateInitialPartitionsAction(metrics, changeStreamDao, endTime);
-    assertEquals(
-        ProcessContinuation.resume(),
-        generateInitialPartitionsAction.run(receiver, tracker, watermarkEstimator, startTime));
+    generateInitialPartitionsAction.run(receiver, startTime);
     verify(receiver, times(2))
         .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
     List<PartitionRecord> actualPartitions = partitionRecordArgumentCaptor.getAllValues();
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java
index 8ff9368cf47..f997db07a8a 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ReadChangeStreamPartitionActionTest.java
@@ -45,6 +45,7 @@ import java.util.Optional;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.ChangeStreamDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.ReadChangeStreamPartitionProgressTracker;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction.StreamProgress;
@@ -94,13 +95,17 @@ public class ReadChangeStreamPartitionActionTest {
     receiver = mock(DoFn.OutputReceiver.class);
     watermarkEstimator = mock(ManualWatermarkEstimator.class);
 
-    partition = ByteStringRange.create("A", "B");
+    partition = ByteStringRange.create("A", "C");
     uuid = "123456";
     Instant startTime = Instant.now();
     Instant parentLowWatermark = Instant.now();
-    partitionRecord = new PartitionRecord(partition, startTime, uuid, parentLowWatermark, null);
+    partitionRecord =
+        new PartitionRecord(
+            partition, startTime, uuid, parentLowWatermark, Collections.emptyList(), null);
     when(tracker.currentRestriction()).thenReturn(restriction);
-    when(metadataTableDao.lockPartition(partition, uuid)).thenReturn(true);
+    when(metadataTableDao.lockAndRecordPartition(partitionRecord)).thenReturn(true);
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(true);
+    when(restriction.isEmpty()).thenReturn(true);
     when(restriction.getCurrentToken()).thenReturn(null);
     when(restriction.getCloseStream()).thenReturn(null);
     // Setting watermark estimator to now so we don't debug.
@@ -126,14 +131,63 @@ public class ReadChangeStreamPartitionActionTest {
         action.run(partitionRecord, tracker, receiver, watermarkEstimator);
 
     assertEquals(DoFn.ProcessContinuation.stop(), result);
-    // Verify that on successful lock, we don't tryClaim on the tracker because we have no outputs.
+    // Verify that on successful lock, we don't tryClaim on the tracker.
     verify(tracker, never()).tryClaim(any());
     verify(changeStreamAction).run(any(), any(), any(), any(), any(), anyBoolean());
   }
 
   @Test
   public void testLockingRowFailsStops() throws IOException {
-    when(metadataTableDao.lockPartition(partition, uuid)).thenReturn(false);
+    when(metadataTableDao.lockAndRecordPartition(partitionRecord)).thenReturn(false);
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
+
+    final DoFn.ProcessContinuation result =
+        action.run(partitionRecord, tracker, receiver, watermarkEstimator);
+
+    assertEquals(DoFn.ProcessContinuation.stop(), result);
+
+    // On failure to lock, we try to claim a fail to lock, so it will terminate gracefully.
+    StreamProgress streamProgress = new StreamProgress();
+    streamProgress.setFailToLock(true);
+    verify(tracker).tryClaim(streamProgress);
+    verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+  }
+
+  @Test
+  public void testLockingRowNotNeededAfterFirstRun() throws IOException {
+    when(metadataTableDao.lockAndRecordPartition(partitionRecord)).thenReturn(false);
+    // After the first run, we don't try to lock but we check if the lock owner is the same.
+    when(restriction.isEmpty()).thenReturn(false);
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(true);
+
+    final ServerStream<ChangeStreamRecord> responses = mock(ServerStream.class);
+    final Iterator<ChangeStreamRecord> responseIterator = mock(Iterator.class);
+    when(responses.iterator()).thenReturn(responseIterator);
+
+    Heartbeat mockHeartBeat = Mockito.mock(Heartbeat.class);
+    when(responseIterator.next()).thenReturn(mockHeartBeat);
+    when(responseIterator.hasNext()).thenReturn(true);
+    when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any(), anyBoolean()))
+        .thenReturn(responses);
+
+    when(changeStreamAction.run(any(), any(), any(), any(), any(), anyBoolean()))
+        .thenReturn(Optional.of(DoFn.ProcessContinuation.stop()));
+
+    final DoFn.ProcessContinuation result =
+        action.run(partitionRecord, tracker, receiver, watermarkEstimator);
+
+    assertEquals(DoFn.ProcessContinuation.stop(), result);
+    // Verify that on successful lock, we don't tryClaim on the tracker.
+    verify(tracker, never()).tryClaim(any());
+    verify(changeStreamAction).run(any(), any(), any(), any(), any(), anyBoolean());
+  }
+
+  @Test
+  public void testLockingRowNotNeededAfterFirstRunNotSame() throws IOException {
+    when(metadataTableDao.lockAndRecordPartition(partitionRecord)).thenReturn(false);
+    // After the first run, we check if the lock is the same, but in this case it's not.
+    when(restriction.isEmpty()).thenReturn(false);
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
 
     final DoFn.ProcessContinuation result =
         action.run(partitionRecord, tracker, receiver, watermarkEstimator);
@@ -174,147 +228,195 @@ public class ReadChangeStreamPartitionActionTest {
 
   @Test
   public void testCloseStreamTerminateOKStatus() throws IOException {
+    // Force lock fail because CloseStream should not depend on locking
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
     CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
     Status statusProto = Status.newBuilder().setCode(0).build();
     Mockito.when(mockCloseStream.getStatus())
         .thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
     when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+    when(restriction.isEmpty()).thenReturn(false);
     final DoFn.ProcessContinuation result =
         action.run(partitionRecord, tracker, receiver, watermarkEstimator);
     assertEquals(DoFn.ProcessContinuation.stop(), result);
     // Should terminate before reaching processing stream partition responses.
     verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+    // Should not try claim any restriction when processing CloseStream
+    verify(tracker, (never())).tryClaim(any());
     // Should decrement the metric on termination.
     verify(metrics).decPartitionStreamCount();
     // Should not try to write any new partition to the metadata table.
-    verify(metadataTableDao, never()).writeNewPartition(any(), any(), any(), any());
+    verify(metadataTableDao, never()).writeNewPartition(any());
+    verify(metadataTableDao, never()).releaseStreamPartitionLockForDeletion(any(), any());
     verify(metadataTableDao, never()).deleteStreamPartitionRow(any());
   }
 
   @Test
   public void testCloseStreamTerminateNotOutOfRangeStatus() throws IOException {
+    // Force lock fail because CloseStream should not depend on locking
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
     // Out of Range code is 11.
     CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
     Status statusProto = Status.newBuilder().setCode(10).build();
     Mockito.when(mockCloseStream.getStatus())
         .thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
     when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+    when(restriction.isEmpty()).thenReturn(false);
     final DoFn.ProcessContinuation result =
         action.run(partitionRecord, tracker, receiver, watermarkEstimator);
     assertEquals(DoFn.ProcessContinuation.stop(), result);
     // Should terminate before reaching processing stream partition responses.
     verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+    // Should not try claim any restriction when processing CloseStream
+    verify(tracker, (never())).tryClaim(any());
     // Should decrement the metric on termination.
     verify(metrics).decPartitionStreamCount();
     // Should not try to write any new partition to the metadata table.
-    verify(metadataTableDao, never()).writeNewPartition(any(), any(), any(), any());
+    verify(metadataTableDao, never()).writeNewPartition(any());
+    verify(metadataTableDao, never()).releaseStreamPartitionLockForDeletion(any(), any());
     verify(metadataTableDao, never()).deleteStreamPartitionRow(any());
   }
 
   @Test
   public void testCloseStreamWritesContinuationTokens() throws IOException {
-    ChangeStreamContinuationToken changeStreamContinuationToken1 =
-        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "AJ"), "1234");
-    ChangeStreamContinuationToken changeStreamContinuationToken2 =
-        ChangeStreamContinuationToken.create(ByteStringRange.create("AJ", "B"), "5678");
+    // Force lock fail because CloseStream should not depend on locking
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
+    ChangeStreamContinuationToken tokenAB =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+    ChangeStreamContinuationToken tokenBC =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
 
     CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
     Status statusProto = Status.newBuilder().setCode(11).build();
     Mockito.when(mockCloseStream.getStatus())
         .thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
     Mockito.when(mockCloseStream.getChangeStreamContinuationTokens())
-        .thenReturn(Arrays.asList(changeStreamContinuationToken1, changeStreamContinuationToken2));
+        .thenReturn(Arrays.asList(tokenAB, tokenBC));
     Mockito.when(mockCloseStream.getNewPartitions())
-        .thenReturn(
-            Arrays.asList(
-                changeStreamContinuationToken1.getPartition(),
-                changeStreamContinuationToken2.getPartition()));
+        .thenReturn(Arrays.asList(tokenAB.getPartition(), tokenBC.getPartition()));
 
     when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+    when(restriction.isEmpty()).thenReturn(false);
     final DoFn.ProcessContinuation result =
         action.run(partitionRecord, tracker, receiver, watermarkEstimator);
     assertEquals(DoFn.ProcessContinuation.stop(), result);
     // Should terminate before reaching processing stream partition responses.
     verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+    // Should not try claim any restriction when processing CloseStream
+    verify(tracker, (never())).tryClaim(any());
     // Should decrement the metric on termination.
     verify(metrics).decPartitionStreamCount();
     // Write the new partitions.
-    verify(metadataTableDao)
-        .writeNewPartition(any(), eq(changeStreamContinuationToken1), any(), any());
-    verify(metadataTableDao)
-        .writeNewPartition(any(), eq(changeStreamContinuationToken2), any(), any());
+    NewPartition newPartitionAB =
+        new NewPartition(
+            tokenAB.getPartition(),
+            Collections.singletonList(tokenAB),
+            watermarkEstimator.getState());
+    verify(metadataTableDao).writeNewPartition(newPartitionAB);
+    NewPartition newPartitionBC =
+        new NewPartition(
+            tokenBC.getPartition(),
+            Collections.singletonList(tokenBC),
+            watermarkEstimator.getState());
+    verify(metadataTableDao).writeNewPartition(newPartitionBC);
+    verify(metadataTableDao, times(1))
+        .releaseStreamPartitionLockForDeletion(
+            partitionRecord.getPartition(), partitionRecord.getUuid());
     verify(metadataTableDao, times(1)).deleteStreamPartitionRow(partitionRecord.getPartition());
   }
 
   @Test
   public void testCloseStreamNewPartitionMerge() throws IOException {
+    // Force lock fail because CloseStream should not depend on locking
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
     // NewPartitions field includes the merge target. ChangeStreamContinuationToken's partition may
     // not be the same as the new partition.
-    // AB merging into AC. The new partition is AC and the corresponding
-    // ChangeStreamContinuationToken is for AB
-    ByteStringRange childPartitionAC = ByteStringRange.create("A", "C");
-    ChangeStreamContinuationToken parentTokenAB =
-        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+    // AC merging into AD. The new partition is AD and the corresponding
+    // ChangeStreamContinuationToken is for AC
+    ByteStringRange childPartitionAD = ByteStringRange.create("A", "D");
+    ChangeStreamContinuationToken parentTokenAC =
+        ChangeStreamContinuationToken.create(partition, "AC");
 
     CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
     Status statusProto = Status.newBuilder().setCode(11).build();
     Mockito.when(mockCloseStream.getStatus())
         .thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
     Mockito.when(mockCloseStream.getChangeStreamContinuationTokens())
-        .thenReturn(Collections.singletonList(parentTokenAB));
+        .thenReturn(Collections.singletonList(parentTokenAC));
     Mockito.when(mockCloseStream.getNewPartitions())
-        .thenReturn(Collections.singletonList(childPartitionAC));
+        .thenReturn(Collections.singletonList(childPartitionAD));
+    when(restriction.isEmpty()).thenReturn(false);
 
     when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+    when(restriction.isEmpty()).thenReturn(false);
     final DoFn.ProcessContinuation result =
         action.run(partitionRecord, tracker, receiver, watermarkEstimator);
     assertEquals(DoFn.ProcessContinuation.stop(), result);
     // Should terminate before reaching processing stream partition responses.
     verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+    // Should not try claim any restriction when processing CloseStream
+    verify(tracker, (never())).tryClaim(any());
     // Should decrement the metric on termination.
     verify(metrics).decPartitionStreamCount();
     // Write the new partitions.
-    // Write the new partitions.
-    verify(metadataTableDao)
-        .writeNewPartition(eq(childPartitionAC), eq(parentTokenAB), eq(partition), any());
-    verify(metadataTableDao, times(1)).deleteStreamPartitionRow(partitionRecord.getPartition());
+    NewPartition newPartitionAD =
+        new NewPartition(
+            childPartitionAD,
+            Collections.singletonList(parentTokenAC),
+            watermarkEstimator.getState());
+    verify(metadataTableDao).writeNewPartition(eq(newPartitionAD));
+    verify(metadataTableDao, times(1))
+        .releaseStreamPartitionLockForDeletion(
+            partitionRecord.getPartition(), partitionRecord.getUuid());
     verify(metadataTableDao, times(1)).deleteStreamPartitionRow(partitionRecord.getPartition());
   }
 
   @Test
   public void testCloseStreamMergeWithoutNewPartitionsField() throws IOException {
+    // Force lock fail because CloseStream should not depend on locking
+    when(metadataTableDao.doHoldLock(partition, uuid)).thenReturn(false);
     // This is testing the old version of the API without the NewPartitions field. This means the
     // ChangeStreamContinuationToken's partition represents the new partition.
-    // AB merging into AC.
-    ByteStringRange childPartitionAC = ByteStringRange.create("A", "C");
+    // AC merging into AD.
+    ByteStringRange childPartitionAD = ByteStringRange.create("A", "D");
     // The partition in the token is different from the test above. The token is actually encoded
-    // for partition AB, but in this version, the partition in the token represents the NEW (child)
+    // for partition AC, but in this version, the partition in the token represents the NEW (child)
     // partition. This has been replaced by the new_partitions field in CloseStream.
-    ChangeStreamContinuationToken parentTokenAB =
-        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "C"), "AB");
+    ChangeStreamContinuationToken parentTokenAC =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "D"), "AC");
 
     CloseStream mockCloseStream = Mockito.mock(CloseStream.class);
     Status statusProto = Status.newBuilder().setCode(11).build();
     Mockito.when(mockCloseStream.getStatus())
         .thenReturn(com.google.cloud.bigtable.common.Status.fromProto(statusProto));
     Mockito.when(mockCloseStream.getChangeStreamContinuationTokens())
-        .thenReturn(Collections.singletonList(parentTokenAB));
+        .thenReturn(Collections.singletonList(parentTokenAC));
     Mockito.when(mockCloseStream.getNewPartitions()).thenReturn(Collections.emptyList());
 
     when(restriction.getCloseStream()).thenReturn(mockCloseStream);
+    when(restriction.isEmpty()).thenReturn(false);
     final DoFn.ProcessContinuation result =
         action.run(partitionRecord, tracker, receiver, watermarkEstimator);
     assertEquals(DoFn.ProcessContinuation.stop(), result);
     // Should terminate before reaching processing stream partition responses.
     verify(changeStreamAction, never()).run(any(), any(), any(), any(), any(), anyBoolean());
+    // Should not try claim any restriction when processing CloseStream
+    verify(tracker, (never())).tryClaim(any());
     // Should decrement the metric on termination.
     verify(metrics).decPartitionStreamCount();
     // We have to correct the partition in the tokens if we don't have new_partitions field.
-    ChangeStreamContinuationToken realTokenAB =
-        getTokenWithCorrectPartition(ByteStringRange.create("A", "B"), parentTokenAB);
+    ChangeStreamContinuationToken realTokenAC =
+        getTokenWithCorrectPartition(partition, parentTokenAC);
     // Write the new partitions.
-    verify(metadataTableDao)
-        .writeNewPartition(eq(childPartitionAC), eq(realTokenAB), eq(partition), any());
+    NewPartition newPartitionAD =
+        new NewPartition(
+            childPartitionAD,
+            Collections.singletonList(realTokenAC),
+            watermarkEstimator.getState());
+    verify(metadataTableDao).writeNewPartition(eq(newPartitionAD));
+    verify(metadataTableDao, times(1))
+        .releaseStreamPartitionLockForDeletion(
+            partitionRecord.getPartition(), partitionRecord.getUuid());
     verify(metadataTableDao, times(1)).deleteStreamPartitionRow(partitionRecord.getPartition());
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java
new file mode 100644
index 00000000000..3f8737c0595
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/action/ResumeFromPreviousPipelineActionTest.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.action;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
+import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
+import com.google.cloud.bigtable.data.v2.BigtableDataClient;
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
+import com.google.protobuf.InvalidProtocolBufferException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.transforms.DoFn.OutputReceiver;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ResumeFromPreviousPipelineActionTest {
+  @ClassRule
+  public static final BigtableEmulatorRule BIGTABLE_EMULATOR_RULE = BigtableEmulatorRule.create();
+
+  private MetadataTableDao metadataTableDao;
+  @Mock private ChangeStreamMetrics metrics;
+  @Mock private OutputReceiver<PartitionRecord> receiver;
+
+  private Instant endTime;
+  private static BigtableDataClient dataClient;
+  private static BigtableTableAdminClient adminClient;
+
+  private ResumeFromPreviousPipelineAction action;
+
+  @Captor ArgumentCaptor<PartitionRecord> partitionRecordArgumentCaptor;
+
+  @BeforeClass
+  public static void beforeClass() throws IOException {
+    BigtableTableAdminSettings adminSettings =
+        BigtableTableAdminSettings.newBuilderForEmulator(BIGTABLE_EMULATOR_RULE.getPort())
+            .setProjectId("fake-project")
+            .setInstanceId("fake-instance")
+            .build();
+    adminClient = BigtableTableAdminClient.create(adminSettings);
+    BigtableDataSettings dataSettingsBuilder =
+        BigtableDataSettings.newBuilderForEmulator(BIGTABLE_EMULATOR_RULE.getPort())
+            .setProjectId("fake-project")
+            .setInstanceId("fake-instance")
+            .build();
+    dataClient = BigtableDataClient.create(dataSettingsBuilder);
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    String changeStreamId = UniqueIdGenerator.generateRowKeyPrefix();
+    MetadataTableAdminDao metadataTableAdminDao =
+        new MetadataTableAdminDao(
+            adminClient, null, changeStreamId, MetadataTableAdminDao.DEFAULT_METADATA_TABLE_NAME);
+    metadataTableAdminDao.createMetadataTable();
+    metadataTableDao =
+        new MetadataTableDao(
+            dataClient,
+            metadataTableAdminDao.getTableId(),
+            metadataTableAdminDao.getChangeStreamNamePrefix());
+    endTime = Instant.now().plus(Duration.standardSeconds(10));
+    ProcessNewPartitionsAction processNewPartitionsAction =
+        new ProcessNewPartitionsAction(metrics, metadataTableDao, endTime);
+    action =
+        new ResumeFromPreviousPipelineAction(
+            metrics, metadataTableDao, endTime, processNewPartitionsAction);
+  }
+
+  @Test
+  public void testResetMissingPartitions() throws InvalidProtocolBufferException {
+    HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
+    missingPartitionsDuration.put(ByteStringRange.create("A", "B"), Instant.now());
+    metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionsDuration);
+    assertFalse(metadataTableDao.readDetectNewPartitionMissingPartitions().isEmpty());
+    action.run(receiver);
+    assertTrue(metadataTableDao.readDetectNewPartitionMissingPartitions().isEmpty());
+  }
+
+  @Test
+  public void testOutputExistingStreamPartitions() throws InvalidProtocolBufferException {
+    ByteStringRange partition1 = ByteStringRange.create("A", "B");
+    Instant watermark1 = Instant.now().minus(Duration.standardSeconds(10));
+    PartitionRecord partitionWithStartTime =
+        new PartitionRecord(partition1, watermark1, "1", watermark1, Collections.emptyList(), null);
+    metadataTableDao.lockAndRecordPartition(partitionWithStartTime);
+
+    ByteStringRange partition2 = ByteStringRange.create("B", "D");
+    ChangeStreamContinuationToken partition2Token1 =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "tokenBC");
+    ChangeStreamContinuationToken partition2Token2 =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("C", "D"), "tokenCD");
+    Instant watermark2 = Instant.now().plus(Duration.standardMinutes(1));
+    PartitionRecord partitionWithInitialTokens =
+        new PartitionRecord(
+            partition2,
+            Arrays.asList(partition2Token1, partition2Token2),
+            "2",
+            watermark2,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionWithInitialTokens);
+
+    ByteStringRange partition3 = ByteStringRange.create("D", "H");
+    Instant watermark3 = Instant.now();
+    PartitionRecord partitionWithContinuationToken =
+        new PartitionRecord(partition3, watermark3, "3", watermark3, Collections.emptyList(), null);
+    metadataTableDao.lockAndRecordPartition(partitionWithContinuationToken);
+    ChangeStreamContinuationToken partition3token =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("D", "H"), "tokenDH");
+    metadataTableDao.updateWatermark(partition3, watermark3, partition3token);
+
+    ByteStringRange partition4 = ByteStringRange.create("H", "I");
+    ChangeStreamContinuationToken partition4token =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("H", "I"), "tokenHI");
+    Instant watermark4 = Instant.now();
+    PartitionRecord unlockedPartition =
+        new PartitionRecord(
+            partition4,
+            Collections.singletonList(partition4token),
+            "4",
+            watermark4,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(unlockedPartition);
+    metadataTableDao.releaseStreamPartitionLockForDeletion(partition4, unlockedPartition.getUuid());
+
+    action.run(receiver);
+    verify(receiver, times(4))
+        .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
+    List<PartitionRecord> actualPartitions = partitionRecordArgumentCaptor.getAllValues();
+
+    PartitionRecord actualPartition1 = actualPartitions.get(0);
+    assertEquals(partition1, actualPartition1.getPartition());
+    assertEquals(partitionWithStartTime.getUuid(), actualPartition1.getUuid());
+    assertNull(actualPartition1.getChangeStreamContinuationTokens());
+    assertEquals(partitionWithStartTime.getStartTime(), actualPartition1.getStartTime());
+    assertEquals(endTime, actualPartition1.getEndTime());
+    assertEquals(
+        partitionWithStartTime.getParentLowWatermark(), actualPartition1.getParentLowWatermark());
+
+    PartitionRecord actualPartition2 = actualPartitions.get(1);
+    assertEquals(partition2, actualPartition2.getPartition());
+    assertEquals(partitionWithInitialTokens.getUuid(), actualPartition2.getUuid());
+    assertNull(actualPartition2.getStartTime());
+    assertEquals(
+        partitionWithInitialTokens.getChangeStreamContinuationTokens(),
+        actualPartition2.getChangeStreamContinuationTokens());
+    assertEquals(endTime, actualPartition2.getEndTime());
+    assertEquals(
+        partitionWithInitialTokens.getParentLowWatermark(),
+        actualPartition2.getParentLowWatermark());
+
+    PartitionRecord actualPartition3 = actualPartitions.get(2);
+    assertEquals(partition3, actualPartition3.getPartition());
+    assertEquals(partitionWithContinuationToken.getUuid(), actualPartition3.getUuid());
+    assertNull(actualPartition3.getStartTime());
+    assertNotNull(actualPartition3.getChangeStreamContinuationTokens());
+    assertEquals(1, actualPartition3.getChangeStreamContinuationTokens().size());
+    assertEquals(partition3token, actualPartition3.getChangeStreamContinuationTokens().get(0));
+    assertEquals(endTime, actualPartition3.getEndTime());
+    assertEquals(
+        partitionWithContinuationToken.getParentLowWatermark(),
+        actualPartition3.getParentLowWatermark());
+
+    PartitionRecord actualPartition4 = actualPartitions.get(3);
+    assertEquals(partition4, actualPartition4.getPartition());
+    // partition4 was unlocked so it gets a new uuid.
+    assertNotEquals("4", actualPartition4.getUuid());
+    assertNull(actualPartition4.getStartTime());
+    assertNotNull(actualPartition4.getChangeStreamContinuationTokens());
+    assertEquals(1, actualPartition4.getChangeStreamContinuationTokens().size());
+    assertEquals(partition4token, actualPartition4.getChangeStreamContinuationTokens().get(0));
+    assertEquals(endTime, actualPartition4.getEndTime());
+    assertEquals(
+        unlockedPartition.getParentLowWatermark(), actualPartition4.getParentLowWatermark());
+  }
+
+  @Test
+  public void testOutputNewPartitions() throws InvalidProtocolBufferException {
+    ByteStringRange partition1 = ByteStringRange.create("A", "B");
+    ChangeStreamContinuationToken token1 =
+        ChangeStreamContinuationToken.create(partition1, "tokenAB");
+    Instant watermark1 = Instant.now().minus(Duration.standardSeconds(10));
+    NewPartition newSplitPartition =
+        new NewPartition(partition1, Collections.singletonList(token1), watermark1);
+    metadataTableDao.writeNewPartition(newSplitPartition);
+
+    ByteStringRange partition2 = ByteStringRange.create("B", "D");
+    ByteStringRange partition2parent1 = ByteStringRange.create("C", "D");
+    ChangeStreamContinuationToken token2 =
+        ChangeStreamContinuationToken.create(partition2parent1, "tokenCD");
+    Instant watermark2 = Instant.now().plus(Duration.standardSeconds(10));
+    NewPartition newPartitionMissingParent =
+        new NewPartition(partition2, Collections.singletonList(token2), watermark2);
+    metadataTableDao.writeNewPartition(newPartitionMissingParent);
+
+    ByteStringRange partition3 = ByteStringRange.create("D", "E");
+    ChangeStreamContinuationToken token3 =
+        ChangeStreamContinuationToken.create(partition3, "tokenDE");
+    Instant watermark3 = Instant.now().plus(Duration.standardSeconds(5));
+    NewPartition deletedNewPartition =
+        new NewPartition(partition3, Collections.singletonList(token3), watermark3);
+    metadataTableDao.writeNewPartition(deletedNewPartition);
+    metadataTableDao.markNewPartitionForDeletion(deletedNewPartition);
+
+    // There are only 2 NewPartition rows because partition3 is deleted.
+    assertEquals(2, metadataTableDao.readNewPartitions().size());
+
+    action.run(receiver);
+    verify(receiver, times(2))
+        .outputWithTimestamp(partitionRecordArgumentCaptor.capture(), eq(Instant.EPOCH));
+    List<PartitionRecord> actualPartitions = partitionRecordArgumentCaptor.getAllValues();
+
+    PartitionRecord actualPartition1 = actualPartitions.get(0);
+    assertEquals(partition1, actualPartition1.getPartition());
+    assertEquals(1, actualPartition1.getChangeStreamContinuationTokens().size());
+    assertEquals(token1, actualPartition1.getChangeStreamContinuationTokens().get(0));
+    assertNull(actualPartition1.getStartTime());
+    assertEquals(watermark1, actualPartition1.getParentLowWatermark());
+    assertEquals(endTime, actualPartition1.getEndTime());
+    assertEquals(
+        Collections.singletonList(newSplitPartition), actualPartition1.getParentPartitions());
+    // Uuid is filled.
+    assertFalse(actualPartition1.getUuid().isEmpty());
+
+    // The 2nd partition is not partition2 but partition3 because partition2 is missing a parent.
+    // Even though partition3 is marked for deletion, we still process it.
+    PartitionRecord actualPartition2 = actualPartitions.get(1);
+    assertEquals(partition3, actualPartition2.getPartition());
+    assertEquals(1, actualPartition2.getChangeStreamContinuationTokens().size());
+    assertEquals(token3, actualPartition2.getChangeStreamContinuationTokens().get(0));
+    assertNull(actualPartition2.getStartTime());
+    assertEquals(watermark3, actualPartition2.getParentLowWatermark());
+    assertEquals(endTime, actualPartition2.getEndTime());
+    assertEquals(
+        Collections.singletonList(deletedNewPartition), actualPartition2.getParentPartitions());
+
+    // There is only 1 NewPartition row now because we processed them except for the NewPartition
+    // with missing parent.
+    assertEquals(1, metadataTableDao.readNewPartitions().size());
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java
index c7f915da638..a98814d4f48 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dao/MetadataTableDaoTest.java
@@ -17,11 +17,15 @@
  */
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao;
 
+import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder.parseWatermarkFromRow;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
-import com.google.api.gax.rpc.ServerStream;
 import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
 import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
 import com.google.cloud.bigtable.data.v2.BigtableDataClient;
@@ -36,10 +40,16 @@ import com.google.protobuf.ByteString;
 import com.google.protobuf.InvalidProtocolBufferException;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.encoder.MetadataTableEncoder;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.StreamPartitionWithWatermark;
+import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -52,6 +62,7 @@ import org.slf4j.LoggerFactory;
 
 @RunWith(JUnit4.class)
 public class MetadataTableDaoTest {
+
   private static final Logger LOG = LoggerFactory.getLogger(MetadataTableDaoTest.class);
 
   @ClassRule
@@ -135,11 +146,13 @@ public class MetadataTableDaoTest {
     ByteString rowKey = metadataTableDao.convertPartitionToStreamPartitionRowKey(partition);
     // Class to try to lock the partition in a separate thread.
     class LockPartition implements Runnable {
-      final String id;
+      final PartitionRecord partitionRecord =
+          new PartitionRecord(
+              partition, Collections.emptyList(), Instant.now(), Collections.emptyList());
       boolean locked = false;
 
       LockPartition(String id) {
-        this.id = id;
+        partitionRecord.setUuid(id);
       }
 
       @Override
@@ -149,7 +162,7 @@ public class MetadataTableDaoTest {
           // race.
           int sleep = (int) (Math.random() * 1000);
           Thread.sleep(sleep);
-          if (metadataTableDao.lockPartition(partition, id)) {
+          if (metadataTableDao.lockAndRecordPartition(partitionRecord)) {
             locked = true;
           }
         } catch (InterruptedException e) {
@@ -212,58 +225,65 @@ public class MetadataTableDaoTest {
 
   @Test
   public void testReadStreamPartitionsWithWatermark() throws InvalidProtocolBufferException {
-    ByteStringRange partitionWithWatermark = ByteStringRange.create("a", "");
+    ByteStringRange lockedPartition = ByteStringRange.create("", "a");
+    PartitionRecord partitionRecord =
+        new PartitionRecord(
+            lockedPartition,
+            Instant.now(),
+            UniqueIdGenerator.getNextId(),
+            Instant.now(),
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord);
+
+    // Only one row has both watermark and is locked.
+    List<StreamPartitionWithWatermark> streamPartitionsWithWatermark =
+        metadataTableDao.readStreamPartitionsWithWatermark();
+    assertEquals(1, streamPartitionsWithWatermark.size());
+    assertEquals(
+        partitionRecord.getParentLowWatermark(),
+        streamPartitionsWithWatermark.get(0).getWatermark());
+    assertEquals(
+        partitionRecord.getPartition(), streamPartitionsWithWatermark.get(0).getPartition());
+
     Instant watermark = Instant.now();
-    metadataTableDao.updateWatermark(partitionWithWatermark, watermark, null);
-
-    // This should only return rows where the watermark has been set, not rows that have been locked
-    // but have not yet set the first watermark
-    ServerStream<Row> rowsWithWatermark =
-        metadataTableDao.readFromMdTableStreamPartitionsWithWatermark();
-    ArrayList<Row> metadataRows = new ArrayList<>();
-    for (Row row : rowsWithWatermark) {
-      metadataRows.add(row);
-    }
-    assertEquals(1, metadataRows.size());
-    Instant metadataWatermark = MetadataTableEncoder.parseWatermarkFromRow(metadataRows.get(0));
-    assertEquals(watermark, metadataWatermark);
-    ByteStringRange rowKeyResponse =
-        metadataTableDao.convertStreamPartitionRowKeyToPartition(metadataRows.get(0).getKey());
-    assertEquals(partitionWithWatermark, rowKeyResponse);
+    // Update the watermark on the locked partition.
+    metadataTableDao.updateWatermark(lockedPartition, watermark, null);
+
+    // Watermark is updated.
+    streamPartitionsWithWatermark = metadataTableDao.readStreamPartitionsWithWatermark();
+    assertEquals(1, streamPartitionsWithWatermark.size());
+    assertEquals(watermark, streamPartitionsWithWatermark.get(0).getWatermark());
   }
 
   @Test
   public void testNewPartitionsWriteRead() throws InvalidProtocolBufferException {
     // This test a split of ["", "") to ["", "a") and ["a", "")
-    ByteStringRange parentPartition = ByteStringRange.create("", "");
-    ByteStringRange partition1 = ByteStringRange.create("", "a");
-    ChangeStreamContinuationToken changeStreamContinuationToken1 =
-        ChangeStreamContinuationToken.create(partition1, "tk1");
-    ByteStringRange partition2 = ByteStringRange.create("a", "");
-    ChangeStreamContinuationToken changeStreamContinuationToken2 =
-        ChangeStreamContinuationToken.create(partition2, "tk2");
-
     Instant lowWatermark = Instant.now();
-    metadataTableDao.writeNewPartition(
-        partition1, changeStreamContinuationToken1, parentPartition, lowWatermark);
-    metadataTableDao.writeNewPartition(
-        partition2, changeStreamContinuationToken2, parentPartition, lowWatermark);
+    ByteStringRange partitionEmptyA = ByteStringRange.create("", "a");
+    ChangeStreamContinuationToken tokenEmptyA =
+        ChangeStreamContinuationToken.create(partitionEmptyA, "EmptyA");
+    NewPartition newPartitionEmptyA =
+        new NewPartition(partitionEmptyA, Collections.singletonList(tokenEmptyA), lowWatermark);
 
-    ServerStream<Row> rows = metadataTableDao.readNewPartitions();
+    ByteStringRange partitionAEmpty = ByteStringRange.create("a", "");
+    ChangeStreamContinuationToken tokenAEmpty =
+        ChangeStreamContinuationToken.create(partitionAEmpty, "AEmpty");
+    NewPartition newPartitionAEmpty =
+        new NewPartition(partitionAEmpty, Collections.singletonList(tokenAEmpty), lowWatermark);
+
+    metadataTableDao.writeNewPartition(newPartitionEmptyA);
+    metadataTableDao.writeNewPartition(newPartitionAEmpty);
+
+    List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
     int rowsCount = 0;
     boolean matchedPartition1 = false;
     boolean matchedPartition2 = false;
-    for (Row row : rows) {
+    for (NewPartition newPartition : newPartitions) {
       rowsCount++;
-      ByteString newPartitionPrefix =
-          metadataTableDao
-              .getChangeStreamNamePrefix()
-              .concat(MetadataTableAdminDao.NEW_PARTITION_PREFIX);
-      ByteStringRange partition =
-          ByteStringRange.toByteStringRange(row.getKey().substring(newPartitionPrefix.size()));
-      if (partition.equals(partition1)) {
+      if (newPartition.getPartition().equals(partitionEmptyA)) {
         matchedPartition1 = true;
-      } else if (partition.equals(partition2)) {
+      } else if (newPartition.getPartition().equals(partitionAEmpty)) {
         matchedPartition2 = true;
       }
     }
@@ -272,34 +292,263 @@ public class MetadataTableDaoTest {
     assertEquals(2, rowsCount);
   }
 
+  @Test
+  public void testMarkNewPartitionForDeletion() throws InvalidProtocolBufferException {
+    ByteStringRange childPartition = ByteStringRange.create("A", "C");
+    ChangeStreamContinuationToken token =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
+    NewPartition newPartition =
+        new NewPartition(childPartition, Collections.singletonList(token), Instant.now());
+    metadataTableDao.writeNewPartition(newPartition);
+    List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+    assertEquals(1, newPartitions.size());
+    assertEquals(childPartition, newPartitions.get(0).getPartition());
+
+    // Once it's marked for deletion, it is not visible to readNewPartitions.
+    metadataTableDao.markNewPartitionForDeletion(newPartitions.get(0));
+    assertEquals(0, metadataTableDao.readNewPartitions().size());
+  }
+
+  @Test
+  public void testMarkNewPartitionForDeletionVisibleAfterOneMinute()
+      throws InvalidProtocolBufferException {
+    ByteStringRange childPartition = ByteStringRange.create("A", "C");
+    ByteStringRange parentPartition = ByteStringRange.create("B", "C");
+    ChangeStreamContinuationToken token =
+        ChangeStreamContinuationToken.create(parentPartition, "BC");
+    NewPartition newPartition =
+        new NewPartition(childPartition, Collections.singletonList(token), Instant.now());
+    metadataTableDao.writeNewPartition(newPartition);
+    List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+    assertEquals(1, newPartitions.size());
+    assertEquals(childPartition, newPartitions.get(0).getPartition());
+
+    // Mark it for deletion but more than 60s ago.
+    RowMutation rowMutation =
+        RowMutation.create(
+                metadataTableAdminDao.getTableId(),
+                metadataTableDao.convertPartitionToNewPartitionRowKey(childPartition))
+            .setCell(
+                MetadataTableAdminDao.CF_SHOULD_DELETE,
+                ByteStringRange.serializeToByteString(parentPartition),
+                Instant.now().minus(Duration.standardSeconds(61)).getMillis() * 1_000L,
+                1);
+    dataClient.mutateRow(rowMutation);
+
+    // We should be able to read back the NewPartition.
+    newPartitions = metadataTableDao.readNewPartitions();
+    assertEquals(1, newPartitions.size());
+    assertEquals(childPartition, newPartitions.get(0).getPartition());
+  }
+
+  @Test
+  public void testDeleteNewPartition() throws InvalidProtocolBufferException {
+    ByteStringRange childPartition = ByteStringRange.create("A", "C");
+    ChangeStreamContinuationToken token =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
+    NewPartition newPartition =
+        new NewPartition(childPartition, Collections.singletonList(token), Instant.now());
+    metadataTableDao.writeNewPartition(newPartition);
+    ChangeStreamContinuationToken token2 =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+    NewPartition newPartition2 =
+        new NewPartition(childPartition, Collections.singletonList(token2), Instant.now());
+    metadataTableDao.writeNewPartition(newPartition2);
+
+    List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+    assertEquals(1, newPartitions.size());
+    assertEquals(childPartition, newPartitions.get(0).getPartition());
+
+    // Did not mark new partition for deletion. Cannot delete.
+    assertFalse(metadataTableDao.deleteNewPartition(newPartitions.get(0)));
+    assertEquals(1, metadataTableDao.readNewPartitions().size());
+
+    metadataTableDao.markNewPartitionForDeletion(newPartitions.get(0));
+    assertTrue(metadataTableDao.deleteNewPartition(newPartitions.get(0)));
+    assertEquals(0, metadataTableDao.readNewPartitions().size());
+  }
+
+  @Test
+  public void testPartialDeleteNewPartition() throws InvalidProtocolBufferException {
+    ByteStringRange childPartition = ByteStringRange.create("A", "C");
+    ChangeStreamContinuationToken token =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
+    NewPartition newPartition =
+        new NewPartition(childPartition, Collections.singletonList(token), Instant.now());
+    metadataTableDao.writeNewPartition(newPartition);
+    ChangeStreamContinuationToken token2 =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+    NewPartition newPartition2 =
+        new NewPartition(childPartition, Collections.singletonList(token2), Instant.now());
+    metadataTableDao.writeNewPartition(newPartition2);
+
+    List<NewPartition> newPartitions = metadataTableDao.readNewPartitions();
+    assertEquals(1, newPartitions.size());
+    assertEquals(childPartition, newPartitions.get(0).getPartition());
+    assertEquals(2, newPartitions.get(0).getChangeStreamContinuationTokens().size());
+
+    // Only mark one of the parent for deletion but try to delete the entire new partition. This
+    // will succeed but only the parent marked for deletion will be deleted.
+    metadataTableDao.markNewPartitionForDeletion(newPartition);
+    assertFalse(metadataTableDao.deleteNewPartition(newPartitions.get(0)));
+    newPartitions = metadataTableDao.readNewPartitions();
+    assertEquals(1, newPartitions.size());
+    assertEquals(childPartition, newPartitions.get(0).getPartition());
+    assertEquals(1, newPartitions.get(0).getChangeStreamContinuationTokens().size());
+    assertEquals(token2, newPartitions.get(0).getChangeStreamContinuationTokens().get(0));
+
+    // Try deleting the parent that isn't marked for deletion. It is not possible
+    assertFalse(metadataTableDao.deleteNewPartition(newPartition2));
+    newPartitions = metadataTableDao.readNewPartitions();
+    assertEquals(1, newPartitions.size());
+    assertEquals(childPartition, newPartitions.get(0).getPartition());
+    assertEquals(1, newPartitions.get(0).getChangeStreamContinuationTokens().size());
+    assertEquals(token2, newPartitions.get(0).getChangeStreamContinuationTokens().get(0));
+
+    // Mark the remaining parent for deletion.
+    metadataTableDao.markNewPartitionForDeletion(newPartition2);
+    assertTrue(metadataTableDao.deleteNewPartition(newPartition2));
+    assertEquals(0, metadataTableDao.readNewPartitions().size());
+  }
+
   @Test
   public void testUpdateAndReadWatermark() throws InvalidProtocolBufferException {
     ByteStringRange partition1 = ByteStringRange.create("a", "b");
     Instant watermark1 = Instant.now();
+
+    PartitionRecord partitionRecord =
+        new PartitionRecord(
+            partition1,
+            Instant.now(),
+            UniqueIdGenerator.getNextId(),
+            Instant.now(),
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord);
     metadataTableDao.updateWatermark(partition1, watermark1, null);
     ByteStringRange partition2 = ByteStringRange.create("b", "c");
     Instant watermark2 = Instant.now();
+    PartitionRecord partitionRecord2 =
+        new PartitionRecord(
+            partition2,
+            Instant.now(),
+            UniqueIdGenerator.getNextId(),
+            Instant.now(),
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord2);
     metadataTableDao.updateWatermark(partition2, watermark2, null);
 
-    ServerStream<Row> rows = metadataTableDao.readFromMdTableStreamPartitionsWithWatermark();
-    int rowsCount = 0;
-    boolean matchedPartition1 = false;
-    boolean matchedPartition2 = false;
-    for (Row row : rows) {
-      rowsCount++;
-      ByteStringRange partition =
-          metadataTableDao.convertStreamPartitionRowKeyToPartition(row.getKey());
-      if (partition.equals(partition1)) {
-        assertEquals(watermark1, MetadataTableEncoder.parseWatermarkFromRow(row));
-        matchedPartition1 = true;
-      } else if (partition.equals(partition2)) {
-        assertEquals(watermark2, MetadataTableEncoder.parseWatermarkFromRow(row));
-        matchedPartition2 = true;
-      }
-    }
-    assertEquals(2, rowsCount);
-    assertTrue(matchedPartition1);
-    assertTrue(matchedPartition2);
+    List<StreamPartitionWithWatermark> streamPartitionsWithWatermark =
+        metadataTableDao.readStreamPartitionsWithWatermark();
+    assertThat(
+        streamPartitionsWithWatermark,
+        containsInAnyOrder(
+            new StreamPartitionWithWatermark(partition1, watermark1),
+            new StreamPartitionWithWatermark(partition2, watermark2)));
+  }
+
+  @Test
+  public void testReadingExistingStreamPartitions() throws InvalidProtocolBufferException {
+    ByteStringRange partition1 = ByteStringRange.create("A", "B");
+    Instant watermark1 = Instant.now();
+    PartitionRecord partitionRecord1 =
+        new PartitionRecord(partition1, watermark1, "1", watermark1, Collections.emptyList(), null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord1);
+
+    ByteStringRange partition2 = ByteStringRange.create("B", "D");
+    ChangeStreamContinuationToken partition2Token1 =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "tokenBC");
+    ChangeStreamContinuationToken partition2Token2 =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("C", "D"), "tokenCD");
+    Instant watermark2 = Instant.now();
+    PartitionRecord partitionRecord2 =
+        new PartitionRecord(
+            partition2,
+            Arrays.asList(partition2Token1, partition2Token2),
+            "2",
+            watermark2,
+            Collections.emptyList(),
+            null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord2);
+
+    List<PartitionRecord> partitionRecords = metadataTableDao.readAllStreamPartitions();
+    assertThat(partitionRecords, containsInAnyOrder(partitionRecord1, partitionRecord2));
+
+    // Update the watermark of partition2
+    Instant watermark3 = watermark2.plus(Duration.standardSeconds(10));
+    metadataTableDao.updateWatermark(partition2, watermark3, null);
+    PartitionRecord partitionRecord2UpdatedWatermark =
+        new PartitionRecord(
+            partition2,
+            Arrays.asList(partition2Token1, partition2Token2),
+            "2",
+            watermark3,
+            Collections.emptyList(),
+            null);
+    partitionRecords = metadataTableDao.readAllStreamPartitions();
+    assertThat(
+        partitionRecords, containsInAnyOrder(partitionRecord1, partitionRecord2UpdatedWatermark));
+
+    // Update partition1 with new watermark and continuation token
+    Instant watermark4 = watermark1.plus(Duration.standardMinutes(1));
+    ChangeStreamContinuationToken partition1Token =
+        ChangeStreamContinuationToken.create(partition1, "tokenAB");
+    metadataTableDao.updateWatermark(partition1, watermark4, partition1Token);
+    PartitionRecord partitionRecord1UpdatedWatermark =
+        new PartitionRecord(
+            partition1,
+            Collections.singletonList(partition1Token),
+            "1",
+            watermark4,
+            Collections.emptyList(),
+            null);
+    partitionRecords = metadataTableDao.readAllStreamPartitions();
+    assertThat(
+        partitionRecords,
+        containsInAnyOrder(partitionRecord1UpdatedWatermark, partitionRecord2UpdatedWatermark));
+
+    // Release the lock on partition2, the results should be the same.
+    metadataTableDao.releaseStreamPartitionLockForDeletion(partition2, partitionRecord2.getUuid());
+    partitionRecords = metadataTableDao.readAllStreamPartitions();
+    PartitionRecord partitionRecord2NoUuid =
+        new PartitionRecord(
+            partition2,
+            Arrays.asList(partition2Token1, partition2Token2),
+            watermark3,
+            Collections.emptyList());
+    assertThat(
+        partitionRecords,
+        containsInAnyOrder(partitionRecord1UpdatedWatermark, partitionRecord2NoUuid));
+
+    // Delete partition2
+    metadataTableDao.deleteStreamPartitionRow(partition2);
+    partitionRecords = metadataTableDao.readAllStreamPartitions();
+    assertThat(partitionRecords, containsInAnyOrder(partitionRecord1UpdatedWatermark));
+  }
+
+  @Test
+  public void testUpdateDetectNewPartitionWatermark() {
+    Instant watermark = Instant.now();
+    metadataTableDao.updateDetectNewPartitionWatermark(watermark);
+    Row row =
+        dataClient.readRow(
+            metadataTableAdminDao.getTableId(),
+            metadataTableDao
+                .getChangeStreamNamePrefix()
+                .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX));
+    assertNull(MetadataTableEncoder.parseTokenFromRow(row));
+    assertEquals(watermark, parseWatermarkFromRow(row));
+  }
+
+  @Test
+  public void testReadWriteLowWatermark() {
+    Instant t = Instant.now();
+    metadataTableDao.updateDetectNewPartitionWatermark(t);
+    assertEquals(t, metadataTableDao.readDetectNewPartitionsState().getWatermark());
+    t = t.plus(Duration.standardMinutes(10));
+    metadataTableDao.updateDetectNewPartitionWatermark(t);
+    assertEquals(t, metadataTableDao.readDetectNewPartitionsState().getWatermark());
   }
 
   @Test
@@ -312,23 +561,23 @@ public class MetadataTableDaoTest {
         dataClient.readRow(
             metadataTableAdminDao.getTableId(),
             metadataTableDao.convertPartitionToStreamPartitionRowKey(partition));
-    assertEquals(token.getToken(), MetadataTableEncoder.getTokenFromRow(row));
-    assertEquals(watermark, MetadataTableEncoder.parseWatermarkFromRow(row));
+    assertEquals(token.getToken(), MetadataTableEncoder.parseTokenFromRow(row));
+    assertEquals(watermark, parseWatermarkFromRow(row));
   }
 
   @Test
   public void readAndWriteValidMissingPartitionsDuration() {
-    HashMap<ByteStringRange, Long> missingPartitionsDuration = new HashMap<>();
-    missingPartitionsDuration.put(ByteStringRange.create("A", "B"), 100L);
+    HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
+    missingPartitionsDuration.put(ByteStringRange.create("A", "B"), Instant.ofEpochMilli(100));
     metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionsDuration);
-    HashMap<ByteStringRange, Long> actualMissingPartitionsDuration =
+    HashMap<ByteStringRange, Instant> actualMissingPartitionsDuration =
         metadataTableDao.readDetectNewPartitionMissingPartitions();
     assertEquals(missingPartitionsDuration, actualMissingPartitionsDuration);
   }
 
   @Test
   public void readAndWriteInvalidMissingPartitionsDuration() {
-    HashMap<ByteStringRange, Long> missingPartitionsDuration = new HashMap<>();
+    HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
 
     RowMutation rowMutation =
         RowMutation.create(
@@ -343,16 +592,104 @@ public class MetadataTableDaoTest {
     dataClient.mutateRow(rowMutation);
 
     // We should still be able to read the invalid serialization and return an empty map.
-    HashMap<ByteStringRange, Long> actualMissingPartitionsDuration =
+    HashMap<ByteStringRange, Instant> actualMissingPartitionsDuration =
         metadataTableDao.readDetectNewPartitionMissingPartitions();
     assertEquals(missingPartitionsDuration, actualMissingPartitionsDuration);
   }
 
   @Test
   public void readMissingPartitionsWithoutDNPRow() {
-    HashMap<ByteStringRange, Long> missingPartitionsDuration = new HashMap<>();
-    HashMap<ByteStringRange, Long> actualMissingPartitionsDuration =
+    HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
+    HashMap<ByteStringRange, Instant> actualMissingPartitionsDuration =
         metadataTableDao.readDetectNewPartitionMissingPartitions();
     assertEquals(missingPartitionsDuration, actualMissingPartitionsDuration);
   }
+
+  @Test
+  public void readMissingPartitionsWithoutMissingPartitionsCell() {
+    HashMap<ByteStringRange, Instant> missingPartitionsDuration = new HashMap<>();
+    // Write DNP row but no missing partitions cell.
+    metadataTableDao.updateDetectNewPartitionWatermark(Instant.now());
+    HashMap<ByteStringRange, Instant> actualMissingPartitionsDuration =
+        metadataTableDao.readDetectNewPartitionMissingPartitions();
+    assertEquals(missingPartitionsDuration, actualMissingPartitionsDuration);
+  }
+
+  @Test
+  public void testReleaseStreamPartitionLock() throws InvalidProtocolBufferException {
+    ByteStringRange partition = ByteStringRange.create("A", "B");
+    String uuid = "1234";
+    PartitionRecord partitionRecord =
+        new PartitionRecord(
+            partition, Instant.now(), uuid, Instant.now(), Collections.emptyList(), null);
+    metadataTableDao.lockAndRecordPartition(partitionRecord);
+    assertFalse(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+    assertTrue(metadataTableDao.doHoldLock(partition, uuid));
+
+    assertFalse(metadataTableDao.releaseStreamPartitionLockForDeletion(partition, "0000"));
+    assertFalse(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+    assertTrue(metadataTableDao.doHoldLock(partition, uuid));
+
+    assertTrue(metadataTableDao.releaseStreamPartitionLockForDeletion(partition, uuid));
+    assertTrue(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+    assertFalse(metadataTableDao.doHoldLock(partition, uuid));
+  }
+
+  @Test
+  public void testDeleteStreamPartitionRow() throws InvalidProtocolBufferException {
+    ByteStringRange partition = ByteStringRange.create("A", "B");
+    String uuid = "1234";
+    PartitionRecord partitionRecord =
+        new PartitionRecord(
+            partition, Instant.now(), uuid, Instant.now(), Collections.emptyList(), null);
+    assertTrue(metadataTableDao.lockAndRecordPartition(partitionRecord));
+    metadataTableDao.updateWatermark(partition, Instant.now(), null);
+    assertFalse(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+    assertTrue(metadataTableDao.doHoldLock(partition, uuid));
+
+    // Can't delete it because lock needs to be released first
+    assertFalse(metadataTableDao.deleteStreamPartitionRow(partition));
+    assertFalse(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+    assertTrue(metadataTableDao.doHoldLock(partition, uuid));
+
+    // Release and delete
+    assertTrue(metadataTableDao.releaseStreamPartitionLockForDeletion(partition, uuid));
+    assertTrue(metadataTableDao.deleteStreamPartitionRow(partition));
+    assertTrue(metadataTableDao.readStreamPartitionsWithWatermark().isEmpty());
+    assertFalse(metadataTableDao.doHoldLock(partition, uuid));
+  }
+
+  @Test
+  public void testLockPartitionRecordsMetadata() {
+    ByteStringRange partition = ByteStringRange.create("A", "C");
+    ChangeStreamContinuationToken token =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "token1");
+    PartitionRecord partitionRecord =
+        new PartitionRecord(
+            partition,
+            Collections.singletonList(token),
+            "1234",
+            Instant.now(),
+            Collections.emptyList(),
+            Instant.now());
+    metadataTableDao.lockAndRecordPartition(partitionRecord);
+    assertTrue(metadataTableDao.doHoldLock(partition, "1234"));
+    // Check that watermark and tokens are correctly written.
+    Row row =
+        dataClient.readRow(
+            metadataTableAdminDao.getTableId(),
+            metadataTableDao.convertPartitionToStreamPartitionRowKey(partition));
+    assertTrue(
+        row.getCells(
+                MetadataTableAdminDao.CF_SHOULD_DELETE, MetadataTableAdminDao.QUALIFIER_DEFAULT)
+            .isEmpty());
+    assertEquals(partitionRecord.getParentLowWatermark(), parseWatermarkFromRow(row));
+    assertEquals(
+        token.toByteString(),
+        row.getCells(
+                MetadataTableAdminDao.CF_INITIAL_TOKEN,
+                ByteStringRange.serializeToByteString(token.getPartition()))
+            .get(0)
+            .getValue());
+  }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java
index 0807719c8cb..a9fa54d4c49 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/InitializeDoFnTest.java
@@ -17,11 +17,11 @@
  */
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.dofn;
 
-import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNull;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -31,14 +31,18 @@ import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
 import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
 import com.google.cloud.bigtable.data.v2.BigtableDataClient;
 import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
-import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
 import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
+import com.google.protobuf.ByteString;
 import java.io.IOException;
+import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO;
+import org.apache.beam.sdk.io.gcp.bigtable.BigtableIO.ExistingPipelineOptions;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.DaoFactory;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.InitialPipelineState;
 import org.apache.beam.sdk.transforms.DoFn;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.primitives.Longs;
+import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -56,7 +60,7 @@ public class InitializeDoFnTest {
   @Mock private DaoFactory daoFactory;
   @Mock private transient MetadataTableAdminDao metadataTableAdminDao;
   private transient MetadataTableDao metadataTableDao;
-  @Mock private DoFn.OutputReceiver<Instant> outputReceiver;
+  @Mock private DoFn.OutputReceiver<InitialPipelineState> outputReceiver;
   private final String tableId = "table";
 
   private static BigtableDataClient dataClient;
@@ -83,6 +87,7 @@ public class InitializeDoFnTest {
     String changeStreamName = "changeStreamName";
     metadataTableAdminDao =
         spy(new MetadataTableAdminDao(adminClient, null, changeStreamName, tableId));
+    metadataTableAdminDao.createMetadataTable();
     doReturn(true).when(metadataTableAdminDao).isAppProfileSingleClusterAndTransactional(any());
     when(daoFactory.getMetadataTableAdminDao()).thenReturn(metadataTableAdminDao);
     metadataTableDao =
@@ -95,29 +100,176 @@ public class InitializeDoFnTest {
   @Test
   public void testInitializeDefault() throws IOException {
     Instant startTime = Instant.now();
-    InitializeDoFn initializeDoFn = new InitializeDoFn(daoFactory, "app-profile", startTime);
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory,
+            "app-profile",
+            startTime,
+            BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS);
     initializeDoFn.processElement(outputReceiver);
-    verify(outputReceiver, times(1)).output(startTime);
-    assertTrue(adminClient.exists(tableId));
-    Row row =
-        dataClient.readRow(
-            tableId,
-            metadataTableAdminDao
-                .getChangeStreamNamePrefix()
-                .concat(MetadataTableAdminDao.DETECT_NEW_PARTITION_SUFFIX));
-    assertNotNull(row);
-    assertEquals(
-        1,
-        row.getCells(MetadataTableAdminDao.CF_VERSION, MetadataTableAdminDao.QUALIFIER_DEFAULT)
-            .size());
-    assertEquals(
-        MetadataTableAdminDao.CURRENT_METADATA_TABLE_VERSION,
-        (int)
-            Longs.fromByteArray(
-                row.getCells(
-                        MetadataTableAdminDao.CF_VERSION, MetadataTableAdminDao.QUALIFIER_DEFAULT)
-                    .get(0)
-                    .getValue()
-                    .toByteArray()));
+    verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+  }
+
+  @Test
+  public void testInitializeStopWithExistingPipeline() throws IOException {
+    metadataTableDao.updateDetectNewPartitionWatermark(Instant.now());
+    Instant startTime = Instant.now();
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory,
+            "app-profile",
+            startTime,
+            BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS);
+    initializeDoFn.processElement(outputReceiver);
+    verify(outputReceiver, never()).output(any());
+  }
+
+  @Test
+  public void testInitializeStopWithoutDNP() throws IOException {
+    // DNP row doesn't exist, so we don't need to stop the pipeline. But some random data row with
+    // the same prefix exists. We want to make sure we clean it up even in "STOP" option.
+    dataClient.mutateRow(
+        RowMutation.create(
+                tableId,
+                metadataTableAdminDao
+                    .getChangeStreamNamePrefix()
+                    .concat(ByteString.copyFromUtf8("existing_row")))
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+    Instant startTime = Instant.now();
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory,
+            "app-profile",
+            startTime,
+            BigtableIO.ExistingPipelineOptions.FAIL_IF_EXISTS);
+    initializeDoFn.processElement(outputReceiver);
+    verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+    assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix()));
+  }
+
+  @Test
+  public void testInitializeNewWithoutDNP() throws IOException {
+    // DNP row doesn't exist, so we output with provided start time and we clean up any existing
+    // rows under the prefix.
+    dataClient.mutateRow(
+        RowMutation.create(
+                tableId,
+                metadataTableAdminDao
+                    .getChangeStreamNamePrefix()
+                    .concat(ByteString.copyFromUtf8("existing_row")))
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+    Instant startTime = Instant.now();
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.NEW);
+    initializeDoFn.processElement(outputReceiver);
+    verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+    assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix()));
+  }
+
+  @Test
+  public void testInitializeNewWithDNP() throws IOException {
+    metadataTableDao.updateDetectNewPartitionWatermark(Instant.now());
+    dataClient.mutateRow(
+        RowMutation.create(
+                tableId,
+                metadataTableAdminDao
+                    .getChangeStreamNamePrefix()
+                    .concat(ByteString.copyFromUtf8("existing_row")))
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+    Instant startTime = Instant.now();
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.NEW);
+    initializeDoFn.processElement(outputReceiver);
+    verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+    assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix()));
+  }
+
+  @Test
+  public void testInitializeResumeWithoutDNP() throws IOException {
+    dataClient.mutateRow(
+        RowMutation.create(
+                tableId,
+                metadataTableAdminDao
+                    .getChangeStreamNamePrefix()
+                    .concat(ByteString.copyFromUtf8("existing_row")))
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+    Instant startTime = Instant.now();
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.RESUME_OR_NEW);
+    initializeDoFn.processElement(outputReceiver);
+    // We want to resume but there's no DNP row, so we resume from the startTime provided.
+    verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+  }
+
+  @Test
+  public void testInitializeResumeWithDNP() throws IOException {
+    Instant resumeTime = Instant.now().minus(Duration.standardSeconds(10000));
+    metadataTableDao.updateDetectNewPartitionWatermark(resumeTime);
+    dataClient.mutateRow(
+        RowMutation.create(
+                tableId,
+                metadataTableAdminDao
+                    .getChangeStreamNamePrefix()
+                    .concat(ByteString.copyFromUtf8("existing_row")))
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+    Instant startTime = Instant.now();
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory, "app-profile", startTime, BigtableIO.ExistingPipelineOptions.RESUME_OR_NEW);
+    initializeDoFn.processElement(outputReceiver);
+    verify(outputReceiver, times(1)).output(new InitialPipelineState(resumeTime, true));
+    assertNull(dataClient.readRow(tableId, metadataTableAdminDao.getChangeStreamNamePrefix()));
+  }
+
+  @Test
+  public void testInitializeSkipCleanupWithoutDNP() throws IOException {
+    ByteString metadataRowKey =
+        metadataTableAdminDao
+            .getChangeStreamNamePrefix()
+            .concat(ByteString.copyFromUtf8("existing_row"));
+    dataClient.mutateRow(
+        RowMutation.create(tableId, metadataRowKey)
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+    Instant startTime = Instant.now();
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory, "app-profile", startTime, ExistingPipelineOptions.SKIP_CLEANUP);
+    initializeDoFn.processElement(outputReceiver);
+    // Skip cleanup will always resume from startTime
+    verify(outputReceiver, times(1)).output(new InitialPipelineState(startTime, false));
+    // Existing metadata shouldn't be cleaned up
+    assertNotNull(dataClient.readRow(tableId, metadataRowKey));
+  }
+
+  @Test
+  public void testInitializeSkipCleanupWithDNP() throws IOException {
+    Instant resumeTime = Instant.now().minus(Duration.standardSeconds(10000));
+    metadataTableDao.updateDetectNewPartitionWatermark(resumeTime);
+    ByteString metadataRowKey =
+        metadataTableAdminDao
+            .getChangeStreamNamePrefix()
+            .concat(ByteString.copyFromUtf8("existing_row"));
+    dataClient.mutateRow(
+        RowMutation.create(tableId, metadataRowKey)
+            .setCell(
+                MetadataTableAdminDao.CF_WATERMARK, MetadataTableAdminDao.QUALIFIER_DEFAULT, 123));
+    Instant startTime = Instant.now();
+    InitializeDoFn initializeDoFn =
+        new InitializeDoFn(
+            daoFactory, "app-profile", startTime, ExistingPipelineOptions.SKIP_CLEANUP);
+    initializeDoFn.processElement(outputReceiver);
+    // We don't want the pipeline to resume to avoid duplicates
+    verify(outputReceiver, never()).output(any());
+    // Existing metadata shouldn't be cleaned up
+    assertNotNull(dataClient.readRow(tableId, metadataRowKey));
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java
index 38aae227787..e8f59c2bd17 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/dofn/ReadChangeStreamPartitionDoFnTest.java
@@ -21,7 +21,6 @@ import static org.apache.beam.sdk.io.gcp.bigtable.changestreams.TimestampConvert
 import static org.junit.Assert.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyObject;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -34,6 +33,7 @@ import com.google.cloud.bigtable.data.v2.models.ChangeStreamRecord;
 import com.google.cloud.bigtable.data.v2.models.Range;
 import com.google.protobuf.ByteString;
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Iterator;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.action.ActionFactory;
@@ -95,8 +95,7 @@ public class ReadChangeStreamPartitionDoFnTest {
   @Test
   public void testProcessElementAndGetSize() throws IOException, InterruptedException {
     long watermarkLag = 10;
-    Instant tenSecondsAgo =
-        Instant.now().minus(org.joda.time.Duration.standardSeconds(watermarkLag));
+    Instant tenSecondsAgo = Instant.now().minus(Duration.standardSeconds(watermarkLag));
     Range.ByteStringRange partitionRange = Range.ByteStringRange.create("", "");
     ChangeStreamContinuationToken testToken =
         ChangeStreamContinuationToken.create(partitionRange, "test");
@@ -106,6 +105,7 @@ public class ReadChangeStreamPartitionDoFnTest {
             tenSecondsAgo,
             "uid-a",
             tenSecondsAgo,
+            Collections.emptyList(),
             Instant.now().plus(Duration.standardSeconds(60)));
     long mutationSize = 100L;
     when(sizeEstimator.sizeOf(any())).thenReturn(mutationSize);
@@ -124,14 +124,13 @@ public class ReadChangeStreamPartitionDoFnTest {
     when(mockMutation.getToken()).thenReturn(testToken.getToken());
     when(mockMutation.getCommitTimestamp()).thenReturn(toThreetenInstant(tenSecondsAgo));
 
-    when(metadataTableDao.lockPartition(any(), any())).thenReturn(true);
+    when(metadataTableDao.lockAndRecordPartition(any())).thenReturn(true);
     ServerStream<ChangeStreamRecord> mockStream = mock(ServerStream.class);
     Iterator<ChangeStreamRecord> mockResponses = mock(Iterator.class);
     when(mockResponses.hasNext()).thenReturn(true, true, true);
     when(mockResponses.next()).thenReturn(mockMutation, mockMutation, mockMutation);
     when(mockStream.iterator()).thenReturn(mockResponses);
-    when(changeStreamDao.readChangeStreamPartition(
-            anyObject(), anyObject(), anyObject(), anyObject(), anyBoolean()))
+    when(changeStreamDao.readChangeStreamPartition(any(), any(), any(), any(), anyBoolean()))
         .thenReturn(mockStream);
 
     when(watermarkEstimator.getState()).thenReturn(tenSecondsAgo);
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleanerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleanerTest.java
new file mode 100644
index 00000000000..27294911b45
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/OrphanedMetadataCleanerTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+import org.junit.Test;
+
+public class OrphanedMetadataCleanerTest {
+
+  @Test
+  public void testOrphanedNewPartitions() {
+    OrphanedMetadataCleaner orphanedMetadataCleaner = new OrphanedMetadataCleaner();
+    // Not waiting long enough
+    NewPartition newPartition1 =
+        new NewPartition(
+            ByteStringRange.create("A", "B"),
+            Collections.emptyList(),
+            Instant.EPOCH,
+            Instant.now().minus(Duration.standardMinutes(10)));
+    orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition1);
+    // Orphaned long enough. Should be cleaned up
+    NewPartition newPartition2 =
+        new NewPartition(
+            ByteStringRange.create("B", "C"),
+            Collections.emptyList(),
+            Instant.EPOCH,
+            Instant.now().minus(Duration.standardMinutes(15)).minus(Duration.standardSeconds(1)));
+    orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition2);
+    // Just on the edge of not orphaned long enough
+    NewPartition newPartition3 =
+        new NewPartition(
+            ByteStringRange.create("C", "D"),
+            Collections.emptyList(),
+            Instant.EPOCH,
+            Instant.now().minus(Duration.standardMinutes(15)).plus(Duration.standardSeconds(1)));
+    orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition3);
+    // Orphaned long enough, but it overlaps with a missing partition, so not cleaned up.
+    NewPartition newPartition4 =
+        new NewPartition(
+            ByteStringRange.create("D", "F"),
+            Collections.emptyList(),
+            Instant.EPOCH,
+            Instant.now().minus(Duration.standardMinutes(15)).minus(Duration.standardSeconds(1)));
+    orphanedMetadataCleaner.addIncompleteNewPartitions(newPartition4);
+    List<ByteStringRange> missingPartitions = new ArrayList<>();
+
+    // Haven't called addMissingPartitions, so we don't know if there are any missing partitions.
+    assertTrue(orphanedMetadataCleaner.getOrphanedNewPartitions().isEmpty());
+
+    missingPartitions.add(ByteStringRange.create("E", "G"));
+    missingPartitions.add(ByteStringRange.create("H", "I"));
+    orphanedMetadataCleaner.addMissingPartitions(missingPartitions);
+
+    assertEquals(
+        Collections.singletonList(newPartition2),
+        orphanedMetadataCleaner.getOrphanedNewPartitions());
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java
index 63c46e6b87f..aab2bdc8f64 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/reconciler/PartitionReconcilerTest.java
@@ -17,41 +17,56 @@
  */
 package org.apache.beam.sdk.io.gcp.bigtable.changestreams.reconciler;
 
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.junit.Assert.assertEquals;
 
 import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
 import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
 import com.google.cloud.bigtable.data.v2.BigtableDataClient;
 import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
+import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
 import com.google.cloud.bigtable.data.v2.models.Range.ByteStringRange;
 import com.google.cloud.bigtable.emulator.v2.BigtableEmulatorRule;
-import com.google.protobuf.ByteString;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Set;
+import java.util.List;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.ChangeStreamMetrics;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.UniqueIdGenerator;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableAdminDao;
 import org.apache.beam.sdk.io.gcp.bigtable.changestreams.dao.MetadataTableDao;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.NewPartition;
+import org.apache.beam.sdk.io.gcp.bigtable.changestreams.model.PartitionRecord;
+import org.joda.time.Duration;
 import org.joda.time.Instant;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.junit.MockitoJUnitRunner;
 
+@RunWith(MockitoJUnitRunner.class)
 public class PartitionReconcilerTest {
   @ClassRule
   public static final BigtableEmulatorRule BIGTABLE_EMULATOR_RULE = BigtableEmulatorRule.create();
 
-  private static final long MORE_THAN_FIVE_MINUTES_MILLI = 5 * 60 * 1000L + 1L;
+  private static final Duration MORE_THAN_ONE_MINUTE = Duration.standardSeconds(61);
+  private static final Duration MORE_THAN_TEN_MINUTES = Duration.standardSeconds(10 * 60 + 1);
 
   private MetadataTableDao metadataTableDao;
 
   private static BigtableDataClient dataClient;
   private static BigtableTableAdminClient adminClient;
 
+  private Instant lowWatermark;
+  private Instant startTime;
+
+  @Mock private ChangeStreamMetrics metrics;
+
   @BeforeClass
   public static void beforeClass() throws IOException {
     BigtableTableAdminSettings adminSettings =
@@ -80,6 +95,10 @@ public class PartitionReconcilerTest {
             dataClient,
             metadataTableAdminDao.getTableId(),
             metadataTableAdminDao.getChangeStreamNamePrefix());
+    lowWatermark = Instant.now();
+    // We set startTime to 30 min prior so if we reconcile without token, we use don't use an hour
+    // prior but the startTime.
+    startTime = lowWatermark.minus(Duration.standardMinutes(30));
   }
 
   // [A-B) is missing because
@@ -89,41 +108,54 @@ public class PartitionReconcilerTest {
   @Test
   public void testNewMissingMergePartitionIsNotReconciled() {
     // No reconciliation should happen because the missing partition hasn't been waiting for more
-    // than 5 minutes.
+    // than 1 minute.
     ByteStringRange partitionAB = ByteStringRange.create("A", "B");
     ByteStringRange partitionAC = ByteStringRange.create("A", "C");
-    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao);
+    ChangeStreamContinuationToken tokenAB = ChangeStreamContinuationToken.create(partitionAB, "AB");
+    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
     partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAB));
     // Since [A-B) received the CloseStream merge, there should be an NewPartitions row for [A-C)
     // with [A-B) as a parent.
-    partitionReconciler.addNewPartition(partitionAC, ByteString.copyFromUtf8("FakeRowKeyForAC"));
-    HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile =
-        partitionReconciler.getPartitionsToReconcile();
-    assertEquals(0, partitionsToReconcile.size());
+    NewPartition newPartitionAC =
+        new NewPartition(partitionAC, Collections.singletonList(tokenAB), Instant.now());
+    partitionReconciler.addIncompleteNewPartitions(newPartitionAC);
+    List<PartitionRecord> reconciledPartitions =
+        partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+    assertEquals(0, reconciledPartitions.size());
   }
 
+  // [A-B) merging into [A-C)
+  // [B-C) did not need to merge into [A-C)
+  // Reconciler should, after 1 minute, output partition [A-B) with continuation token.
   @Test
   public void testLongMissingMergePartitionIsReconciled() {
     ByteStringRange partitionAB = ByteStringRange.create("A", "B");
     ByteStringRange partitionAC = ByteStringRange.create("A", "C");
+    ChangeStreamContinuationToken tokenAB = ChangeStreamContinuationToken.create(partitionAB, "AB");
 
-    // Artificially create that partitionAB has been missing for more than 5 minutes.
-    HashMap<ByteStringRange, Long> missingPartitionDurations = new HashMap<>();
-    missingPartitionDurations.put(
-        partitionAB, Instant.now().getMillis() - MORE_THAN_FIVE_MINUTES_MILLI);
+    // Artificially create that partitionAB has been missing for more than 1 minute.
+    HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+    missingPartitionDurations.put(partitionAB, Instant.now().minus(MORE_THAN_ONE_MINUTE));
     metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
 
-    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao);
+    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
     partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAB));
     // Since [A-B) received the CloseStream merge, there should be an NewPartitions row for [A-C)
     // with [A-B) as a parent.
-    partitionReconciler.addNewPartition(partitionAC, ByteString.copyFromUtf8("FakeRowKeyForAC"));
-    HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile =
-        partitionReconciler.getPartitionsToReconcile();
-    assertEquals(partitionsToReconcile.keySet(), Collections.singleton(partitionAB));
-    assertEquals(
-        partitionsToReconcile.get(partitionAB),
-        Collections.singleton(ByteString.copyFromUtf8("FakeRowKeyForAC")));
+    NewPartition newPartitionAC =
+        new NewPartition(partitionAC, Collections.singletonList(tokenAB), Instant.now());
+    partitionReconciler.addIncompleteNewPartitions(newPartitionAC);
+    List<PartitionRecord> reconciledPartitions =
+        partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+
+    PartitionRecord expectedRecord =
+        new PartitionRecord(
+            partitionAB,
+            Collections.singletonList(tokenAB),
+            lowWatermark,
+            Collections.singletonList(newPartitionAC));
+    assertEquals(1, reconciledPartitions.size());
+    assertEquals(expectedRecord, reconciledPartitions.get(0));
   }
 
   // [A-B) merges into [A-C)
@@ -134,26 +166,47 @@ public class PartitionReconcilerTest {
     ByteStringRange partitionAC = ByteStringRange.create("A", "C");
     ByteStringRange partitionAD = ByteStringRange.create("A", "D");
 
-    // Artificially create that partitionAD has been missing for more than 5 minutes.
-    HashMap<ByteStringRange, Long> missingPartitionDurations = new HashMap<>();
-    missingPartitionDurations.put(
-        partitionAD, Instant.now().getMillis() - MORE_THAN_FIVE_MINUTES_MILLI);
+    ChangeStreamContinuationToken tokenAB =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+    ChangeStreamContinuationToken tokenBC =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
+    ChangeStreamContinuationToken tokenCD =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("C", "D"), "CD");
+    // Artificially create that partitionAD has been missing for more than 10 minutes.
+    HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+    missingPartitionDurations.put(partitionAD, Instant.now().minus(MORE_THAN_TEN_MINUTES));
     metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
 
-    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao);
+    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
     // We are not missing A-B, B-C, C-D, from the metadata table view, we are simply missing A-D.
     partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAD));
-    partitionReconciler.addNewPartition(partitionAC, ByteString.copyFromUtf8("FakeRowKeyForAC"));
-    partitionReconciler.addNewPartition(partitionAD, ByteString.copyFromUtf8("FakeRowKeyForAD"));
-    HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile =
-        partitionReconciler.getPartitionsToReconcile();
-    assertEquals(partitionsToReconcile.keySet(), Collections.singleton(partitionAD));
-    assertEquals(
-        partitionsToReconcile.get(partitionAD),
-        new HashSet<>(
-            Arrays.asList(
-                ByteString.copyFromUtf8("FakeRowKeyForAC"),
-                ByteString.copyFromUtf8("FakeRowKeyForAD"))));
+    // AB merging into AC
+    NewPartition newPartitionAC =
+        new NewPartition(partitionAC, Collections.singletonList(tokenAB), Instant.now());
+    partitionReconciler.addIncompleteNewPartitions(newPartitionAC);
+    // BC and CD merging into AD
+    NewPartition newPartitionAD =
+        new NewPartition(partitionAD, Arrays.asList(tokenBC, tokenCD), Instant.now());
+    partitionReconciler.addIncompleteNewPartitions(newPartitionAD);
+
+    List<PartitionRecord> reconciledPartitions =
+        partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+    // As part of reconciling we split up the parents into their own NewPartition, so they can be
+    // cleaned up separately.
+    NewPartition newPartitionADWithBC =
+        new NewPartition(
+            partitionAD, Collections.singletonList(tokenBC), newPartitionAD.getLowWatermark());
+    NewPartition newPartitionADWithCD =
+        new NewPartition(
+            partitionAD, Collections.singletonList(tokenCD), newPartitionAD.getLowWatermark());
+    PartitionRecord expectedRecord =
+        new PartitionRecord(
+            partitionAD,
+            Arrays.asList(tokenAB, tokenBC, tokenCD),
+            lowWatermark,
+            Arrays.asList(newPartitionAC, newPartitionADWithBC, newPartitionADWithCD));
+    assertEquals(1, reconciledPartitions.size());
+    assertEquals(expectedRecord, reconciledPartitions.get(0));
   }
 
   // [A-B) merges into [A-D)
@@ -161,35 +214,123 @@ public class PartitionReconcilerTest {
   // We're missing [A-C).
   // [A-B) and [B-D) were supposed to merge into [A-D)
   // Before [B-D) received the CloseStream merge, [A-D) actually splits into [A-C) and [C-D) which
-  // is why [B-D) received CloseStream split into [A-C) and [C-D)
-  // Now we need to merge [A-B) and [B-D) into [A-C). [C-D) part of the split was successful on its
-  // own.
+  // is why [B-D) received CloseStream split into [A-C) and [C-D). Note that the tokens in the
+  // CloseStream are for BC and CD. So when performing reconciliation, we're reconciling BC, not BD.
+  // Reconciler should merge [A-B) and [B-C) into [A-C). [C-D) part of the split was successful.
   @Test
   public void testMismatchedMergeSplitPartitionIsReconciled() {
     ByteStringRange partitionAC = ByteStringRange.create("A", "C");
     ByteStringRange partitionAD = ByteStringRange.create("A", "D");
+    ChangeStreamContinuationToken tokenAB =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("A", "B"), "AB");
+    ChangeStreamContinuationToken tokenBC =
+        ChangeStreamContinuationToken.create(ByteStringRange.create("B", "C"), "BC");
 
-    // Artificially create that partitionAC has been missing for more than 5 minutes.
-    HashMap<ByteStringRange, Long> missingPartitionDurations = new HashMap<>();
-    missingPartitionDurations.put(
-        partitionAC, Instant.now().getMillis() - MORE_THAN_FIVE_MINUTES_MILLI);
+    // Artificially create that partitionAC has been missing for more than 10 minutes.
+    HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+    missingPartitionDurations.put(partitionAC, Instant.now().minus(MORE_THAN_TEN_MINUTES));
     metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
 
-    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao);
-    // We are not missing A-B, B-D, from the metadata table view, we are simply missing A-C.
+    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
+    // We are not missing A-B, B-C, from the metadata table view, we are simply missing A-C.
     partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAC));
     // A-B tried to merge into A-D
-    partitionReconciler.addNewPartition(partitionAD, ByteString.copyFromUtf8("FakeRowKeyForAD"));
-    // B-D tried to split/merge into A-C
-    partitionReconciler.addNewPartition(partitionAC, ByteString.copyFromUtf8("FakeRowKeyForAC"));
-    HashMap<ByteStringRange, Set<ByteString>> partitionsToReconcile =
-        partitionReconciler.getPartitionsToReconcile();
-    assertEquals(partitionsToReconcile.keySet(), Collections.singleton(partitionAC));
-    assertEquals(
-        partitionsToReconcile.get(partitionAC),
-        new HashSet<>(
-            Arrays.asList(
-                ByteString.copyFromUtf8("FakeRowKeyForAD"),
-                ByteString.copyFromUtf8("FakeRowKeyForAC"))));
+    NewPartition newPartitionAD =
+        new NewPartition(partitionAD, Collections.singletonList(tokenAB), Instant.now());
+    partitionReconciler.addIncompleteNewPartitions(newPartitionAD);
+    // B-C tried to split/merge into A-C
+    NewPartition newPartitionAC =
+        new NewPartition(partitionAC, Collections.singletonList(tokenBC), Instant.now());
+    partitionReconciler.addIncompleteNewPartitions(newPartitionAC);
+    List<PartitionRecord> reconciledPartitions =
+        partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+
+    PartitionRecord expectedRecord =
+        new PartitionRecord(
+            partitionAC,
+            Arrays.asList(tokenAB, tokenBC),
+            lowWatermark,
+            Arrays.asList(newPartitionAD, newPartitionAC));
+    assertEquals(1, reconciledPartitions.size());
+    assertEquals(expectedRecord, reconciledPartitions.get(0));
+  }
+
+  // AB is missing without any tokens for more than 10 minutes, reconciler will restart the
+  // partition with low watermark - 1 hour.
+  @Test
+  public void testMissingPartitionWithoutToken() {
+    ByteStringRange partitionAB = ByteStringRange.create("A", "B");
+
+    // Artificially create that partitionAB has been missing for more than 10 minutes.
+    HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+    missingPartitionDurations.put(partitionAB, Instant.now().minus(MORE_THAN_TEN_MINUTES));
+    metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
+
+    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
+    partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAB));
+
+    List<PartitionRecord> reconciledPartitions =
+        partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+
+    PartitionRecord expectedRecord =
+        new PartitionRecord(partitionAB, startTime, lowWatermark, Collections.emptyList());
+    assertEquals(1, reconciledPartitions.size());
+    assertEquals(expectedRecord, reconciledPartitions.get(0));
+  }
+
+  // We're missing AD but we only have partition AB and CD. We should reconcile by outputting AB and
+  // CD and then create a new partition for BC with start_time = low watermark - 1 hour
+  @Test
+  public void testMissingPartitionWithSomeToken() {
+    ByteStringRange partitionAD = ByteStringRange.create("A", "D");
+    ByteStringRange partitionBC = ByteStringRange.create("B", "C");
+    ByteStringRange partitionAB = ByteStringRange.create("A", "B");
+    ChangeStreamContinuationToken tokenAB = ChangeStreamContinuationToken.create(partitionAB, "AB");
+    ByteStringRange partitionCD = ByteStringRange.create("C", "D");
+    ChangeStreamContinuationToken tokenCD = ChangeStreamContinuationToken.create(partitionCD, "CD");
+
+    // Artificially create that partitionAD has been missing for more than 10 minutes.
+    HashMap<ByteStringRange, Instant> missingPartitionDurations = new HashMap<>();
+    missingPartitionDurations.put(partitionAD, Instant.now().minus(MORE_THAN_TEN_MINUTES));
+    metadataTableDao.writeDetectNewPartitionMissingPartitions(missingPartitionDurations);
+
+    PartitionReconciler partitionReconciler = new PartitionReconciler(metadataTableDao, metrics);
+    partitionReconciler.addMissingPartitions(Collections.singletonList(partitionAD));
+
+    NewPartition newPartitionAD =
+        new NewPartition(partitionAD, Arrays.asList(tokenAB, tokenCD), Instant.now());
+    partitionReconciler.addIncompleteNewPartitions(newPartitionAD);
+
+    List<PartitionRecord> reconciledPartitions =
+        partitionReconciler.getPartitionsToReconcile(lowWatermark, startTime);
+
+    // As part of reconciling we split up the parents into their own NewPartition, so they can be
+    // cleaned up separately.
+    NewPartition newPartitionADWithAB =
+        new NewPartition(
+            partitionAD, Collections.singletonList(tokenAB), newPartitionAD.getLowWatermark());
+    NewPartition newPartitionADWithCD =
+        new NewPartition(
+            partitionAD, Collections.singletonList(tokenCD), newPartitionAD.getLowWatermark());
+    PartitionRecord expectedRecordAB =
+        new PartitionRecord(
+            partitionAB,
+            Collections.singletonList(tokenAB),
+            lowWatermark,
+            Collections.singletonList(newPartitionADWithAB));
+    PartitionRecord expectedRecordCD =
+        new PartitionRecord(
+            partitionCD,
+            Collections.singletonList(tokenCD),
+            lowWatermark,
+            Collections.singletonList(newPartitionADWithCD));
+    PartitionRecord expectedRecordBC =
+        new PartitionRecord(partitionBC, startTime, lowWatermark, Collections.emptyList());
+    assertEquals(3, reconciledPartitions.size());
+
+    assertThat(
+        reconciledPartitions,
+        containsInAnyOrder(
+            Arrays.asList(expectedRecordAB, expectedRecordBC, expectedRecordCD).toArray()));
   }
 }
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTrackerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTrackerTest.java
new file mode 100644
index 00000000000..42200e8c359
--- /dev/null
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/DetectNewPartitionsTrackerTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.gcp.bigtable.changestreams.restriction;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.beam.sdk.io.range.OffsetRange;
+import org.apache.beam.sdk.transforms.splittabledofn.SplitResult;
+import org.junit.Test;
+
+public class DetectNewPartitionsTrackerTest {
+
+  @Test
+  public void testTrySplit() {
+    final DetectNewPartitionsTracker tracker = new DetectNewPartitionsTracker(0);
+    assertNull(tracker.trySplit(.7));
+    assertThrows(IllegalStateException.class, tracker::checkDone);
+
+    assertTrue(tracker.tryClaim(tracker.currentRestriction().getFrom()));
+    SplitResult<OffsetRange> checkpointResult = tracker.trySplit(0);
+    tracker.checkDone(); // should not throw exception
+    assertEquals(0L, checkpointResult.getPrimary().getFrom());
+    assertEquals(1L, checkpointResult.getPrimary().getTo());
+    assertEquals(1L, checkpointResult.getResidual().getFrom());
+    assertEquals(Long.MAX_VALUE, checkpointResult.getResidual().getTo());
+  }
+}
diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java
index f75986bc419..b0bc8de7c81 100644
--- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java
+++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/changestreams/restriction/ReadChangeStreamPartitionProgressTrackerTest.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import com.google.cloud.bigtable.data.v2.models.ChangeStreamContinuationToken;
 import com.google.cloud.bigtable.data.v2.models.Range;
@@ -54,7 +55,7 @@ public class ReadChangeStreamPartitionProgressTrackerTest {
         tracker.currentRestriction().getEstimatedLowWatermark());
     try {
       tracker.checkDone();
-      assertFalse("Should not reach here because checkDone should have thrown an exception", false);
+      fail("Should not reach here because checkDone should have thrown an exception");
     } catch (IllegalStateException e) {
       assertTrue("There's more work to be done. CheckDone threw an exception", true);
     }
@@ -98,6 +99,6 @@ public class ReadChangeStreamPartitionProgressTrackerTest {
     streamProgress.setFailToLock(false);
     ReadChangeStreamPartitionProgressTracker tracker =
         new ReadChangeStreamPartitionProgressTracker(streamProgress);
-    assertThrows(java.lang.IllegalStateException.class, tracker::checkDone);
+    assertThrows(IllegalStateException.class, tracker::checkDone);
   }
 }