You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by ni...@apache.org on 2015/04/15 00:03:53 UTC
[1/5] samza git commit: Update committer list
Repository: samza
Updated Branches:
refs/heads/samza-sql f8309b2a6 -> bc5833b0d
Update committer list
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/bc5833b0
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/bc5833b0
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/bc5833b0
Branch: refs/heads/samza-sql
Commit: bc5833b0dfa146be5dc50db205350a9f372c17c8
Parents: bf5af14
Author: Yi Pan (Data Infrastructure) <yi...@linkedin.com>
Authored: Tue Apr 14 10:57:36 2015 -0700
Committer: Yi Pan (Data Infrastructure) <yi...@linkedin.com>
Committed: Tue Apr 14 14:40:44 2015 -0700
----------------------------------------------------------------------
docs/community/committers.md | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/bc5833b0/docs/community/committers.md
----------------------------------------------------------------------
diff --git a/docs/community/committers.md b/docs/community/committers.md
index b2dfc90..ca29104 100644
--- a/docs/community/committers.md
+++ b/docs/community/committers.md
@@ -63,3 +63,8 @@ Committer, and PMC member<br/>
Committer, and PMC member<br/>
<a href="https://www.linkedin.com/pub/chinmay-soman/5/610/35" target="_blank"><i class="fa fa-linkedin committer-icon"></i></a>
<a href="https://twitter.com/ChinmaySoman" target="_blank"><i class="fa fa-twitter committer-icon"></i></a>
+
+**Yi Pan**<br/>
+Committer<br/>
+<a href="https://www.linkedin.com/pub/yi-pan/9/85a/238" target="_blank"><i class="fa fa-linkedin committer-icon"></i></a>
+<a href="https://twitter.com/nickpan47" target="_blank"><i class="fa fa-twitter committer-icon"></i></a>
[5/5] samza git commit: SAMZA-629: add instructions for upgrading
websites when releasing new version
Posted by ni...@apache.org.
SAMZA-629: add instructions for upgrading websites when releasing new version
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/233837e5
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/233837e5
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/233837e5
Branch: refs/heads/samza-sql
Commit: 233837e55907dfe3e5dba4b2db6efb642b68876d
Parents: 5589ede
Author: Yan Fang <ya...@gmail.com>
Authored: Thu Apr 9 15:03:49 2015 -0700
Committer: Yi Pan (Data Infrastructure) <yi...@linkedin.com>
Committed: Tue Apr 14 14:40:44 2015 -0700
----------------------------------------------------------------------
docs/README.md | 30 +++++++++++++++++++++++++++++-
1 file changed, 29 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/233837e5/docs/README.md
----------------------------------------------------------------------
diff --git a/docs/README.md b/docs/README.md
index 235fe18..63f03e7 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -52,10 +52,38 @@ To auto-generate the latest Javadocs, run:
bin/generate-javadocs.sh
-## Release
+## Publish
To build and publish the website to Samza's Apache SVN repository, run:
bin/publish-site.sh "updating welcome page" criccomini
This command will re-build the Javadocs and website, checkout https://svn.apache.org/repos/asf/samza/site/ locally, copy the site into the directory, and commit the changes.
+
+## Release-new-version Website Checklist
+
+Assume we want to release x.x.x , need to work on two branches: x.x.x and master.
+
+Following can be done when updating the gradle.properties file
+
+1. in x.x.x branch,
+
+ * modify the docs/_config.yml to make the "version" and "latest-release" to x.x.x
+
+ * remove "git checkout latest" in docs/startup/hello-samza/versioned/index.md
+
+2. in master branch,
+
+ * modify the docs/_config.yml to make the "latest-release" to x.x.x
+
+ * add the x.x.x release to Archive category in docs/_layout/default.html and x.x.x release part in docs/archive/index.html
+
+ * update the download page to use x.x.x release
+
+ * update the version number in "tar -xvf ./target/hello-samza-x.x.x-dist.tar.gz -C deploy/samza" in docs/startup/hello-samza/versioned/index.md
+
+After apache mirrors pick up the new release,
+
+3. in x.x.x branch, run bin/publish-site.sh "updating welcome page" criccomini
+
+4. in master branch, run bin/publish-site.sh "updating welcome page" criccomini
[4/5] samza git commit: SAMZA-646: Remove support for JDK6
Posted by ni...@apache.org.
SAMZA-646: Remove support for JDK6
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/bf5af142
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/bf5af142
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/bf5af142
Branch: refs/heads/samza-sql
Commit: bf5af142fe71441779af0bc8ad83874143dc322a
Parents: f44f8da
Author: Jakob Homan <jg...@gmail.com>
Authored: Fri Apr 10 12:51:53 2015 -0700
Committer: Yi Pan (Data Infrastructure) <yi...@linkedin.com>
Committed: Tue Apr 14 14:40:44 2015 -0700
----------------------------------------------------------------------
bin/check-all.sh | 2 +-
docs/contribute/tests.md | 5 +----
2 files changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/bf5af142/bin/check-all.sh
----------------------------------------------------------------------
diff --git a/bin/check-all.sh b/bin/check-all.sh
index 3e237af..0725b82 100755
--- a/bin/check-all.sh
+++ b/bin/check-all.sh
@@ -22,7 +22,7 @@
set -e
SCALAs=( "2.10" )
-JDKs=( "JAVA6_HOME" "JAVA7_HOME" "JAVA8_HOME" )
+JDKs=( "JAVA7_HOME" "JAVA8_HOME" )
YARNs=( "2.4.0" "2.5.0" )
# get base directory
http://git-wip-us.apache.org/repos/asf/samza/blob/bf5af142/docs/contribute/tests.md
----------------------------------------------------------------------
diff --git a/docs/contribute/tests.md b/docs/contribute/tests.md
index bda13d0..f485ce2 100644
--- a/docs/contribute/tests.md
+++ b/docs/contribute/tests.md
@@ -47,16 +47,13 @@ To run Samza's unit tests against all permutations, run:
bin/check-all.sh
-When run on Linux, this command requires you to set three environment variables:
+When run on Linux, this command requires you to set two environment variables:
- JAVA6_HOME is not set.
JAVA7_HOME is not set.
JAVA8_HOME is not set.
On Mac, check-all.sh will default to the appropriate path for each environment variable if it's not already set:
- JAVA6_HOME is not set.
- JAVA6_HOME defaulted to /System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Home
JAVA7_HOME is not set.
JAVA7_HOME defaulted to /Library/Java/JavaVirtualMachines/jdk1.7.0_67.jdk/Contents/Home
JAVA8_HOME is not set.
[2/5] samza git commit: SAMZA-608;
don't hange on serde errors in system consumers
Posted by ni...@apache.org.
SAMZA-608; don't hange on serde errors in system consumers
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/f44f8dac
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/f44f8dac
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/f44f8dac
Branch: refs/heads/samza-sql
Commit: f44f8dac253eb34348140b5449a2f6a5b29b20db
Parents: 233837e
Author: Yi Pan <ni...@gmail.com>
Authored: Mon Apr 13 10:46:31 2015 -0700
Committer: Yi Pan (Data Infrastructure) <yi...@linkedin.com>
Committed: Tue Apr 14 14:40:44 2015 -0700
----------------------------------------------------------------------
.../apache/samza/system/SystemConsumers.scala | 18 ++++++++++----
.../samza/system/TestSystemConsumers.scala | 25 ++++++++++++++++++--
2 files changed, 37 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/f44f8dac/samza-core/src/main/scala/org/apache/samza/system/SystemConsumers.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/system/SystemConsumers.scala b/samza-core/src/main/scala/org/apache/samza/system/SystemConsumers.scala
index 76fa4ad..1ec5e32 100644
--- a/samza-core/src/main/scala/org/apache/samza/system/SystemConsumers.scala
+++ b/samza-core/src/main/scala/org/apache/samza/system/SystemConsumers.scala
@@ -204,9 +204,7 @@ class SystemConsumers(
metrics.choseObject.inc
metrics.systemStreamMessagesChosen(envelopeFromChooser.getSystemStreamPartition.getSystemStream).inc
- if (!update(systemStreamPartition)) {
- emptySystemStreamPartitionsBySystem.get(systemStreamPartition.getSystem).add(systemStreamPartition)
- }
+ tryUpdate(systemStreamPartition)
}
if (envelopeFromChooser == null || lastPollMs < clock() - pollIntervalMs) {
@@ -257,7 +255,7 @@ class SystemConsumers(
// Update the chooser if it needs a message for this SSP.
if (emptySystemStreamPartitionsBySystem.get(systemStreamPartition.getSystem).remove(systemStreamPartition)) {
- update(systemStreamPartition)
+ tryUpdate(systemStreamPartition)
}
}
}
@@ -266,6 +264,18 @@ class SystemConsumers(
}
}
+ private def tryUpdate(ssp: SystemStreamPartition) {
+ var updated = false
+ try {
+ updated = update(ssp)
+ } finally {
+ if (!updated) {
+ // if failed to update the chooser, add the ssp back into the emptySystemStreamPartitionBySystem map to ensure that we will poll for the next message
+ emptySystemStreamPartitionsBySystem.get(ssp.getSystem).add(ssp)
+ }
+ }
+ }
+
private def refresh {
trace("Refreshing chooser with new messages.")
http://git-wip-us.apache.org/repos/asf/samza/blob/f44f8dac/samza-core/src/test/scala/org/apache/samza/system/TestSystemConsumers.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/test/scala/org/apache/samza/system/TestSystemConsumers.scala b/samza-core/src/test/scala/org/apache/samza/system/TestSystemConsumers.scala
index 3fdc781..fbaa8ee 100644
--- a/samza-core/src/test/scala/org/apache/samza/system/TestSystemConsumers.scala
+++ b/samza-core/src/test/scala/org/apache/samza/system/TestSystemConsumers.scala
@@ -241,9 +241,10 @@ class TestSystemConsumers {
// it should not throw exceptions when deserializaion fails if dropDeserializationError is set to true
val consumers2 = new SystemConsumers(msgChooser, consumer, serdeManager, dropDeserializationError = true)
consumers2.register(systemStreamPartition, "0")
- consumers2.start
consumer(system).putBytesMessage
consumer(system).putStringMessage
+ consumer(system).putBytesMessage
+ consumers2.start
var notThrowException = true;
try {
@@ -251,9 +252,29 @@ class TestSystemConsumers {
} catch {
case e: Throwable => notThrowException = false
}
-
assertTrue("it should not throw any exception", notThrowException)
+
+ var msgEnvelope = Some(consumers2.choose)
+ assertTrue("Consumer did not succeed in receiving the second message after Serde exception in choose", msgEnvelope.get != null)
+ consumers2.stop
+
+ // ensure that the system consumer will continue after poll() method ignored a Serde exception
+ consumer(system).putStringMessage
+ consumer(system).putBytesMessage
+
+ notThrowException = true;
+ try {
+ consumers2.start
+ } catch {
+ case e: Throwable => notThrowException = false
+ }
+ assertTrue("SystemConsumer start should not throw any Serde exception", notThrowException)
+
+ msgEnvelope = null
+ msgEnvelope = Some(consumers2.choose)
+ assertTrue("Consumer did not succeed in receiving the second message after Serde exception in poll", msgEnvelope.get != null)
consumers2.stop
+
}
/**
[3/5] samza git commit: SAMZA-540: Expose checkpointed offset in
OffsetManager
Posted by ni...@apache.org.
SAMZA-540: Expose checkpointed offset in OffsetManager
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/5589edec
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/5589edec
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/5589edec
Branch: refs/heads/samza-sql
Commit: 5589edecffb01c1d493556613bf9e955c88b6861
Parents: f8309b2
Author: Renato MarroquĂn Mogrovejo <re...@gmail.com>
Authored: Thu Apr 9 15:00:38 2015 -0700
Committer: Yi Pan (Data Infrastructure) <yi...@linkedin.com>
Committed: Tue Apr 14 14:40:44 2015 -0700
----------------------------------------------------------------------
docs/startup/hello-samza/versioned/index.md | 2 +-
.../apache/samza/checkpoint/OffsetManager.scala | 17 +++++++---
.../samza/checkpoint/OffsetManagerMetrics.scala | 34 ++++++++++++++++++++
.../apache/samza/container/SamzaContainer.scala | 4 ++-
.../samza/checkpoint/TestOffsetManager.scala | 26 +++++++++++++++
5 files changed, 77 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/5589edec/docs/startup/hello-samza/versioned/index.md
----------------------------------------------------------------------
diff --git a/docs/startup/hello-samza/versioned/index.md b/docs/startup/hello-samza/versioned/index.md
index 507ceb5..b4ab23a 100644
--- a/docs/startup/hello-samza/versioned/index.md
+++ b/docs/startup/hello-samza/versioned/index.md
@@ -53,7 +53,7 @@ Before you can run a Samza job, you need to build a package for it. This package
{% highlight bash %}
mvn clean package
mkdir -p deploy/samza
-tar -xvf ./target/hello-samza-0.9.0-dist.tar.gz -C deploy/samza
+tar -xvf ./target/hello-samza-0.10.0-dist.tar.gz -C deploy/samza
{% endhighlight %}
### Run a Samza Job
http://git-wip-us.apache.org/repos/asf/samza/blob/5589edec/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala
index a40c87f..85c1749 100644
--- a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala
+++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManager.scala
@@ -71,7 +71,8 @@ object OffsetManager extends Logging {
systemStreamMetadata: Map[SystemStream, SystemStreamMetadata],
config: Config,
checkpointManager: CheckpointManager = null,
- systemAdmins: Map[String, SystemAdmin] = Map()) = {
+ systemAdmins: Map[String, SystemAdmin] = Map(),
+ offsetManagerMetrics : OffsetManagerMetrics = new OffsetManagerMetrics) = {
debug("Building offset manager for %s." format systemStreamMetadata)
@@ -98,7 +99,7 @@ object OffsetManager extends Logging {
// Build OffsetSetting so we can create a map for OffsetManager.
(systemStream, OffsetSetting(systemStreamMetadata, defaultOffsetType, resetOffset))
}.toMap
- new OffsetManager(offsetSettings, checkpointManager, systemAdmins)
+ new OffsetManager(offsetSettings, checkpointManager, systemAdmins, offsetManagerMetrics)
}
}
@@ -136,7 +137,12 @@ class OffsetManager(
* SystemAdmins that are used to get next offsets from last checkpointed
* offsets. Map is from system name to SystemAdmin class for the system.
*/
- val systemAdmins: Map[String, SystemAdmin] = Map()) extends Logging {
+ val systemAdmins: Map[String, SystemAdmin] = Map(),
+
+ /**
+ * offsetManagerMetrics for keeping track of checkpointed offsets of each SystemStreamPartition.
+ */
+ val offsetManagerMetrics : OffsetManagerMetrics = new OffsetManagerMetrics ) extends Logging {
/**
* Last offsets processed for each SystemStreamPartition.
@@ -158,6 +164,8 @@ class OffsetManager(
def register(taskName: TaskName, systemStreamPartitionsToRegister: Set[SystemStreamPartition]) {
systemStreamPartitions.getOrElseUpdate(taskName, mutable.Set[SystemStreamPartition]()).addAll(systemStreamPartitionsToRegister)
+ // register metrics
+ systemStreamPartitions.foreach{ case (taskName, ssp) => ssp.foreach (ssp => offsetManagerMetrics.addCheckpointedOffset(ssp, "")) }
}
def start {
@@ -204,6 +212,7 @@ class OffsetManager(
val partitionOffsets = lastProcessedOffsets.filterKeys(sspsForTaskName.contains(_))
checkpointManager.writeCheckpoint(taskName, new Checkpoint(partitionOffsets))
+ lastProcessedOffsets.foreach{ case (ssp, checkpoint) => offsetManagerMetrics.checkpointedOffsets(ssp).set(checkpoint) }
} else {
debug("Skipping checkpointing for taskName %s because no checkpoint manager is defined." format taskName)
}
@@ -225,7 +234,6 @@ class OffsetManager(
private def registerCheckpointManager {
if (checkpointManager != null) {
debug("Registering checkpoint manager.")
-
systemStreamPartitions.keys.foreach(checkpointManager.register)
} else {
debug("Skipping checkpoint manager registration because no manager was defined.")
@@ -249,6 +257,7 @@ class OffsetManager(
if (!shouldKeep) {
info("Ignoring previously checkpointed offset %s for %s since the offset is for a stream that is not currently an input stream." format (offset, systemStreamPartition))
}
+ info("Checkpointed offset is currently %s for %s." format (offset, systemStreamPartition))
shouldKeep
}
} else {
http://git-wip-us.apache.org/repos/asf/samza/blob/5589edec/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManagerMetrics.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManagerMetrics.scala b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManagerMetrics.scala
new file mode 100644
index 0000000..b6219ca
--- /dev/null
+++ b/samza-core/src/main/scala/org/apache/samza/checkpoint/OffsetManagerMetrics.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.checkpoint
+
+import org.apache.samza.metrics._
+import java.util.concurrent.ConcurrentHashMap
+
+import org.apache.samza.system.SystemStreamPartition
+
+class OffsetManagerMetrics(val registry: MetricsRegistry = new MetricsRegistryMap) extends MetricsHelper {
+
+ val checkpointedOffsets = new ConcurrentHashMap[SystemStreamPartition, Gauge[String]]
+
+ def addCheckpointedOffset(systemStreamPartition: SystemStreamPartition, checkpointedOffset: String) {
+ checkpointedOffsets.put(systemStreamPartition, newGauge("%s-%s-%d-checkpointed-offset" format (systemStreamPartition.getSystem, systemStreamPartition.getStream, systemStreamPartition.getPartition.getPartitionId), checkpointedOffset))
+ }
+}
http://git-wip-us.apache.org/repos/asf/samza/blob/5589edec/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
index 66640fe..720fbdc 100644
--- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
+++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
@@ -61,6 +61,7 @@ import org.apache.samza.serializers.model.SamzaObjectMapper
import org.apache.samza.config.JobConfig.Config2Job
import java.lang.Thread.UncaughtExceptionHandler
import org.apache.samza.serializers._
+import org.apache.samza.checkpoint.OffsetManagerMetrics
object SamzaContainer extends Logging {
def main(args: Array[String]) {
@@ -147,6 +148,7 @@ object SamzaContainer extends Logging {
val samzaContainerMetrics = new SamzaContainerMetrics(containerName, registry)
val systemProducersMetrics = new SystemProducersMetrics(registry)
val systemConsumersMetrics = new SystemConsumersMetrics(registry)
+ val offsetManagerMetrics = new OffsetManagerMetrics(registry)
val inputSystemStreamPartitions = containerModel
.getTasks
@@ -336,7 +338,7 @@ object SamzaContainer extends Logging {
info("Got checkpoint manager: %s" format checkpointManager)
- val offsetManager = OffsetManager(inputStreamMetadata, config, checkpointManager, systemAdmins)
+ val offsetManager = OffsetManager(inputStreamMetadata, config, checkpointManager, systemAdmins, offsetManagerMetrics)
info("Got offset manager: %s" format offsetManager)
http://git-wip-us.apache.org/repos/asf/samza/blob/5589edec/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala b/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala
index d18d4c4..a281e79 100644
--- a/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala
+++ b/samza-core/src/test/scala/org/apache/samza/checkpoint/TestOffsetManager.scala
@@ -87,6 +87,32 @@ class TestOffsetManager {
}
@Test
+ def testGetCheckpointedOffsetMetric{
+ val taskName = new TaskName("c")
+ val systemStream = new SystemStream("test-system", "test-stream")
+ val partition = new Partition(0)
+ val systemStreamPartition = new SystemStreamPartition(systemStream, partition)
+ val testStreamMetadata = new SystemStreamMetadata(systemStream.getStream, Map(partition -> new SystemStreamPartitionMetadata("0", "1", "2")))
+ val systemStreamMetadata = Map(systemStream -> testStreamMetadata)
+ val config = new MapConfig
+ val checkpointManager = getCheckpointManager(systemStreamPartition, taskName)
+ val systemAdmins = Map("test-system" -> getSystemAdmin)
+ val offsetManager = OffsetManager(systemStreamMetadata, config, checkpointManager, systemAdmins)
+ offsetManager.register(taskName, Set(systemStreamPartition))
+ offsetManager.start
+ // Should get offset 45 back from the checkpoint manager, which is last processed, and system admin should return 46 as starting offset.
+ offsetManager.checkpoint(taskName)
+ assertEquals("45", offsetManager.offsetManagerMetrics.checkpointedOffsets.get(systemStreamPartition).getValue)
+ offsetManager.update(systemStreamPartition, "46")
+ offsetManager.update(systemStreamPartition, "47")
+ offsetManager.checkpoint(taskName)
+ assertEquals("47", offsetManager.offsetManagerMetrics.checkpointedOffsets.get(systemStreamPartition).getValue)
+ offsetManager.update(systemStreamPartition, "48")
+ offsetManager.checkpoint(taskName)
+ assertEquals("48", offsetManager.offsetManagerMetrics.checkpointedOffsets.get(systemStreamPartition).getValue)
+ }
+
+ @Test
def testShouldResetStreams {
val taskName = new TaskName("c")
val systemStream = new SystemStream("test-system", "test-stream")