You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by lh...@apache.org on 2020/02/06 23:31:27 UTC
[samza] 05/06: SAMZA-2446: Invoke onCheckpoint only for registered
SSPs (#1260)
This is an automated email from the ASF dual-hosted git repository.
lhaiesp pushed a commit to branch 1.3.1
in repository https://gitbox.apache.org/repos/asf/samza.git
commit 964252aed6fb16b7952b93e56c3a0cd8f1072861
Author: bkonold <bk...@users.noreply.github.com>
AuthorDate: Tue Jan 28 15:43:59 2020 -0800
SAMZA-2446: Invoke onCheckpoint only for registered SSPs (#1260)
---
.../org/apache/samza/checkpoint/OffsetManager.scala | 17 +++++++++++------
.../org/apache/samza/checkpoint/TestOffsetManager.scala | 10 +++++++++-
2 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala
index 33fca8f..442d83f 100644
--- a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala
+++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala
@@ -353,13 +353,18 @@ class OffsetManager(
}
}
- // invoke checkpoint listeners
- checkpoint.getOffsets.asScala.groupBy { case (ssp, _) => ssp.getSystem }.foreach {
- case (systemName:String, offsets: Map[SystemStreamPartition, String]) => {
- // Option is empty if there is no checkpointListener for this systemName
- checkpointListeners.get(systemName).foreach(_.onCheckpoint(offsets.asJava))
+ // Invoke checkpoint listeners only for SSPs that are registered with the OffsetManager. For example,
+ // changelog SSPs are not registered but may be present in the Checkpoint if transactional state checkpointing
+ // is enabled.
+ val registeredSSPs = systemStreamPartitions.getOrElse(taskName, Set[SystemStreamPartition]())
+ checkpoint.getOffsets.asScala
+ .filterKeys(registeredSSPs.contains)
+ .groupBy { case (ssp, _) => ssp.getSystem }.foreach {
+ case (systemName:String, offsets: Map[SystemStreamPartition, String]) => {
+ // Option is empty if there is no checkpointListener for this systemName
+ checkpointListeners.get(systemName).foreach(_.onCheckpoint(offsets.asJava))
+ }
}
- }
}
// delete corresponding startpoints after checkpoint is supposed to be committed
diff --git a/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala b/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala
index 50c793c..677504d 100644
--- a/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala
+++ b/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala
@@ -388,9 +388,11 @@ class TestOffsetManager {
val systemName2 = "test-system2"
val systemStream = new SystemStream(systemName, "test-stream")
val systemStream2 = new SystemStream(systemName2, "test-stream2")
+ val systemStream3 = new SystemStream(systemName, "test-stream3")
val partition = new Partition(0)
val systemStreamPartition = new SystemStreamPartition(systemStream, partition)
val systemStreamPartition2 = new SystemStreamPartition(systemStream2, partition)
+ val unregisteredSystemStreamPartition = new SystemStreamPartition(systemStream3, partition)
val testStreamMetadata = new SystemStreamMetadata(systemStream.getStream, Map(partition -> new SystemStreamPartitionMetadata("0", "1", "2")).asJava)
val testStreamMetadata2 = new SystemStreamMetadata(systemStream2.getStream, Map(partition -> new SystemStreamPartitionMetadata("0", "1", "2")).asJava)
val systemStreamMetadata = Map(systemStream -> testStreamMetadata, systemStream2->testStreamMetadata2)
@@ -420,7 +422,10 @@ class TestOffsetManager {
offsetManager.start
// Should get offset 45 back from the checkpoint manager, which is last processed, and system admin should return 46 as starting offset.
assertTrue(startpointManagerUtil.getStartpointManager.getFanOutForTask(taskName).containsKey(systemStreamPartition))
- checkpoint(offsetManager, taskName)
+ val offsetsToCheckpoint = new java.util.HashMap[SystemStreamPartition, String]()
+ offsetsToCheckpoint.putAll(offsetManager.buildCheckpoint(taskName).getOffsets)
+ offsetsToCheckpoint.put(unregisteredSystemStreamPartition, "50")
+ offsetManager.writeCheckpoint(taskName, new Checkpoint(offsetsToCheckpoint))
intercept[IllegalStateException] {
// StartpointManager should stop after last fan out is removed
@@ -434,6 +439,9 @@ class TestOffsetManager {
assertEquals("45", consumer.recentCheckpoint.get(systemStreamPartition))
// make sure only the system with the callbacks gets them
assertNull(consumer.recentCheckpoint.get(systemStreamPartition2))
+ // even though systemStream and systemStream3 share the same checkpointListener, callback should not execute for
+ // systemStream3 since it is not registered with the OffsetManager
+ assertNull(consumer.recentCheckpoint.get(unregisteredSystemStreamPartition))
offsetManager.update(taskName, systemStreamPartition, "46")
offsetManager.update(taskName, systemStreamPartition, "47")