You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ch...@apache.org on 2024/03/08 19:54:45 UTC
(kafka) branch trunk updated: KAFKA-14589 [4/4] Tests of ConsoleGroupCommand rewritten in java (#15465)
This is an automated email from the ASF dual-hosted git repository.
chia7712 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 414365979e9 KAFKA-14589 [4/4] Tests of ConsoleGroupCommand rewritten in java (#15465)
414365979e9 is described below
commit 414365979e960d0705f955579836faff6881f46a
Author: Nikolay <ni...@apache.org>
AuthorDate: Fri Mar 8 22:54:39 2024 +0300
KAFKA-14589 [4/4] Tests of ConsoleGroupCommand rewritten in java (#15465)
Reviewers: Chia-Ping Tsai <ch...@gmail.com>
---
.../kafka/admin/ConsumerGroupCommandTest.scala | 228 --------
.../kafka/admin/ResetConsumerGroupOffsetTest.scala | 514 ------------------
.../consumer/group/ConsumerGroupCommandTest.java | 5 +-
.../group/ResetConsumerGroupOffsetTest.java | 589 +++++++++++++++++++++
4 files changed, 590 insertions(+), 746 deletions(-)
diff --git a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
deleted file mode 100644
index f682df1f1dc..00000000000
--- a/core/src/test/scala/unit/kafka/admin/ConsumerGroupCommandTest.scala
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package kafka.admin
-
-import java.time.Duration
-import java.util.concurrent.{ExecutorService, Executors, TimeUnit}
-import java.util.{Collections, Properties, stream}
-import kafka.admin.ConsumerGroupCommand.{ConsumerGroupCommandOptions, ConsumerGroupService}
-import kafka.api.BaseConsumerTest
-import kafka.integration.KafkaServerTestHarness
-import kafka.server.KafkaConfig
-import kafka.utils.TestUtils
-import org.apache.kafka.clients.admin.AdminClientConfig
-import org.apache.kafka.clients.consumer.{Consumer, ConsumerConfig, GroupProtocol, KafkaConsumer, RangeAssignor}
-import org.apache.kafka.common.{PartitionInfo, TopicPartition}
-import org.apache.kafka.common.errors.WakeupException
-import org.apache.kafka.common.serialization.StringDeserializer
-import org.junit.jupiter.api.{AfterEach, BeforeEach, TestInfo}
-import org.junit.jupiter.params.provider.Arguments
-
-import scala.jdk.CollectionConverters._
-import scala.collection.mutable.ArrayBuffer
-
-class ConsumerGroupCommandTest extends KafkaServerTestHarness {
- import ConsumerGroupCommandTest._
-
- val topic = "foo"
- val group = "test.group"
-
- private var consumerGroupService: List[ConsumerGroupService] = List()
- private var consumerGroupExecutors: List[AbstractConsumerGroupExecutor] = List()
-
- // configure the servers and clients
- override def generateConfigs = {
- val configs = TestUtils.createBrokerConfigs(1, zkConnectOrNull, enableControlledShutdown = false)
-
- if (isNewGroupCoordinatorEnabled()) {
- configs.foreach(_.setProperty(KafkaConfig.NewGroupCoordinatorEnableProp, "true"))
- }
-
- configs.map(KafkaConfig.fromProps)
- }
-
- @BeforeEach
- override def setUp(testInfo: TestInfo): Unit = {
- super.setUp(testInfo)
- createTopic(topic, 1, 1)
- }
-
- @AfterEach
- override def tearDown(): Unit = {
- consumerGroupService.foreach(_.close())
- consumerGroupExecutors.foreach(_.shutdown())
- super.tearDown()
- }
-
- def committedOffsets(topic: String = topic, group: String = group): collection.Map[TopicPartition, Long] = {
- val consumer = createNoAutoCommitConsumer(group)
- try {
- val partitions: Set[TopicPartition] = consumer.partitionsFor(topic)
- .asScala.toSet.map {partitionInfo : PartitionInfo => new TopicPartition(partitionInfo.topic, partitionInfo.partition)}
- consumer.committed(partitions.asJava).asScala.filter(_._2 != null).map { case (k, v) => k -> v.offset }
- } finally {
- consumer.close()
- }
- }
-
- def createNoAutoCommitConsumer(group: String): Consumer[String, String] = {
- val props = new Properties
- props.put("bootstrap.servers", bootstrapServers())
- props.put("group.id", group)
- props.put("enable.auto.commit", "false")
- new KafkaConsumer(props, new StringDeserializer, new StringDeserializer)
- }
-
- def getConsumerGroupService(args: Array[String]): ConsumerGroupService = {
- val opts = new ConsumerGroupCommandOptions(args)
- val service = new ConsumerGroupService(opts, Map(AdminClientConfig.RETRIES_CONFIG -> Int.MaxValue.toString))
- consumerGroupService = service :: consumerGroupService
- service
- }
-
- def addConsumerGroupExecutor(numConsumers: Int,
- topic: String = topic,
- group: String = group,
- strategy: String = classOf[RangeAssignor].getName,
- remoteAssignor: Option[String] = None,
- customPropsOpt: Option[Properties] = None,
- syncCommit: Boolean = false,
- groupProtocol: String = GroupProtocol.CLASSIC.toString): ConsumerGroupExecutor = {
- val executor = new ConsumerGroupExecutor(bootstrapServers(), numConsumers, group, groupProtocol, topic, strategy, remoteAssignor, customPropsOpt, syncCommit)
- addExecutor(executor)
- executor
- }
-
- def addSimpleGroupExecutor(partitions: Iterable[TopicPartition] = Seq(new TopicPartition(topic, 0)),
- group: String = group): SimpleConsumerGroupExecutor = {
- val executor = new SimpleConsumerGroupExecutor(bootstrapServers(), group, partitions)
- addExecutor(executor)
- executor
- }
-
- private def addExecutor(executor: AbstractConsumerGroupExecutor): AbstractConsumerGroupExecutor = {
- consumerGroupExecutors = executor :: consumerGroupExecutors
- executor
- }
-
-}
-
-object ConsumerGroupCommandTest {
- def getTestQuorumAndGroupProtocolParametersAll(): stream.Stream[Arguments] = BaseConsumerTest.getTestQuorumAndGroupProtocolParametersAll()
- def getTestQuorumAndGroupProtocolParametersClassicGroupProtocolOnly(): stream.Stream[Arguments] = BaseConsumerTest.getTestQuorumAndGroupProtocolParametersClassicGroupProtocolOnly()
- def getTestQuorumAndGroupProtocolParametersConsumerGroupProtocolOnly(): stream.Stream[Arguments] = BaseConsumerTest.getTestQuorumAndGroupProtocolParametersConsumerGroupProtocolOnly()
-
- abstract class AbstractConsumerRunnable(broker: String, groupId: String, customPropsOpt: Option[Properties] = None,
- syncCommit: Boolean = false) extends Runnable {
- val props = new Properties
- configure(props)
- customPropsOpt.foreach(props.asScala ++= _.asScala)
- val consumer = new KafkaConsumer(props)
-
- def configure(props: Properties): Unit = {
- props.put("bootstrap.servers", broker)
- props.put("group.id", groupId)
- props.put("key.deserializer", classOf[StringDeserializer].getName)
- props.put("value.deserializer", classOf[StringDeserializer].getName)
- }
-
- def subscribe(): Unit
-
- def run(): Unit = {
- try {
- subscribe()
- while (true) {
- consumer.poll(Duration.ofMillis(Long.MaxValue))
- if (syncCommit)
- consumer.commitSync()
- }
- } catch {
- case _: WakeupException => // OK
- } finally {
- consumer.close()
- }
- }
-
- def shutdown(): Unit = {
- consumer.wakeup()
- }
- }
-
- class ConsumerRunnable(broker: String, groupId: String, groupProtocol: String, topic: String, strategy: String,
- remoteAssignor: Option[String], customPropsOpt: Option[Properties] = None, syncCommit: Boolean = false)
- extends AbstractConsumerRunnable(broker, groupId, customPropsOpt, syncCommit) {
-
- override def configure(props: Properties): Unit = {
- super.configure(props)
- props.put(ConsumerConfig.GROUP_PROTOCOL_CONFIG, groupProtocol)
- if (groupProtocol.toUpperCase == GroupProtocol.CONSUMER.toString) {
- remoteAssignor.foreach { assignor =>
- props.put(ConsumerConfig.GROUP_REMOTE_ASSIGNOR_CONFIG, assignor)
- }
- } else {
- props.put("partition.assignment.strategy", strategy)
- }
- }
-
- override def subscribe(): Unit = {
- consumer.subscribe(Collections.singleton(topic))
- }
- }
-
- class SimpleConsumerRunnable(broker: String, groupId: String, partitions: Iterable[TopicPartition])
- extends AbstractConsumerRunnable(broker, groupId) {
-
- override def subscribe(): Unit = {
- consumer.assign(partitions.toList.asJava)
- }
- }
-
- class AbstractConsumerGroupExecutor(numThreads: Int) {
- private val executor: ExecutorService = Executors.newFixedThreadPool(numThreads)
- private val consumers = new ArrayBuffer[AbstractConsumerRunnable]()
-
- def submit(consumerThread: AbstractConsumerRunnable): Unit = {
- consumers += consumerThread
- executor.submit(consumerThread)
- }
-
- def shutdown(): Unit = {
- consumers.foreach(_.shutdown())
- executor.shutdown()
- executor.awaitTermination(5000, TimeUnit.MILLISECONDS)
- }
- }
-
- class ConsumerGroupExecutor(broker: String, numConsumers: Int, groupId: String, groupProtocol: String, topic: String, strategy: String,
- remoteAssignor: Option[String], customPropsOpt: Option[Properties] = None, syncCommit: Boolean = false)
- extends AbstractConsumerGroupExecutor(numConsumers) {
-
- for (_ <- 1 to numConsumers) {
- submit(new ConsumerRunnable(broker, groupId, groupProtocol, topic, strategy, remoteAssignor, customPropsOpt, syncCommit))
- }
-
- }
-
- class SimpleConsumerGroupExecutor(broker: String, groupId: String, partitions: Iterable[TopicPartition])
- extends AbstractConsumerGroupExecutor(1) {
-
- submit(new SimpleConsumerRunnable(broker, groupId, partitions))
- }
-
-}
-
diff --git a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala b/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
deleted file mode 100644
index 4aafedf0ab3..00000000000
--- a/core/src/test/scala/unit/kafka/admin/ResetConsumerGroupOffsetTest.scala
+++ /dev/null
@@ -1,514 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE
- * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
- * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
- * License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
- * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- */
-package kafka.admin
-
-import java.io.{BufferedWriter, FileWriter}
-import java.text.{SimpleDateFormat}
-import java.util.{Calendar, Date, Properties}
-
-import joptsimple.OptionException
-import kafka.admin.ConsumerGroupCommand.ConsumerGroupService
-import kafka.server.KafkaConfig
-import kafka.utils.TestUtils
-import org.apache.kafka.clients.producer.ProducerRecord
-import org.apache.kafka.common.TopicPartition
-import org.apache.kafka.test
-import org.junit.jupiter.api.Assertions._
-import org.junit.jupiter.api.Test
-
-import scala.jdk.CollectionConverters._
-import scala.collection.Seq
-
-
-
-/**
- * Test cases by:
- * - Non-existing consumer group
- * - One for each scenario, with scope=all-topics
- * - scope=one topic, scenario=to-earliest
- * - scope=one topic+partitions, scenario=to-earliest
- * - scope=topics, scenario=to-earliest
- * - scope=topics+partitions, scenario=to-earliest
- * - export/import
- */
-class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
-
- val overridingProps = new Properties()
- val topic1 = "foo1"
- val topic2 = "foo2"
-
- override def generateConfigs: Seq[KafkaConfig] = {
- TestUtils.createBrokerConfigs(1, zkConnect, enableControlledShutdown = false)
- .map(KafkaConfig.fromProps(_, overridingProps))
- }
-
- private def basicArgs: Array[String] = {
- Array("--reset-offsets",
- "--bootstrap-server", bootstrapServers(),
- "--timeout", test.TestUtils.DEFAULT_MAX_WAIT_MS.toString)
- }
-
- private def buildArgsForGroups(groups: Seq[String], args: String*): Array[String] = {
- val groupArgs = groups.flatMap(group => Seq("--group", group)).toArray
- basicArgs ++ groupArgs ++ args
- }
-
- private def buildArgsForGroup(group: String, args: String*): Array[String] = {
- buildArgsForGroups(Seq(group), args: _*)
- }
-
- private def buildArgsForAllGroups(args: String*): Array[String] = {
- basicArgs ++ Array("--all-groups") ++ args
- }
-
- @Test
- def testResetOffsetsNotExistingGroup(): Unit = {
- val group = "missing.group"
- val args = buildArgsForGroup(group, "--all-topics", "--to-current", "--execute")
- val consumerGroupCommand = getConsumerGroupService(args)
- // Make sure we got a coordinator
- TestUtils.waitUntilTrue(() => {
- consumerGroupCommand.collectGroupState(group).coordinator.host() == "localhost"
- }, "Can't find a coordinator")
- val resetOffsets = consumerGroupCommand.resetOffsets()(group)
- assertEquals(Map.empty, resetOffsets)
- assertEquals(resetOffsets, committedOffsets(group = group))
- }
-
- @Test
- def testResetOffsetsExistingTopic(): Unit = {
- val group = "new.group"
- val args = buildArgsForGroup(group, "--topic", topic, "--to-offset", "50")
- produceMessages(topic, 100)
- resetAndAssertOffsets(args, expectedOffset = 50, dryRun = true)
- resetAndAssertOffsets(args ++ Array("--dry-run"), expectedOffset = 50, dryRun = true)
- resetAndAssertOffsets(args ++ Array("--execute"), expectedOffset = 50)
- }
-
- @Test
- def testResetOffsetsExistingTopicSelectedGroups(): Unit = {
- produceMessages(topic, 100)
- val groups =
- for (id <- 1 to 3) yield {
- val group = this.group + id
- val executor = addConsumerGroupExecutor(numConsumers = 1, topic = topic, group = group)
- awaitConsumerProgress(count = 100L, group = group)
- executor.shutdown()
- group
- }
- val args = buildArgsForGroups(groups,"--topic", topic, "--to-offset", "50")
- resetAndAssertOffsets(args, expectedOffset = 50, dryRun = true)
- resetAndAssertOffsets(args ++ Array("--dry-run"), expectedOffset = 50, dryRun = true)
- resetAndAssertOffsets(args ++ Array("--execute"), expectedOffset = 50)
- }
-
- @Test
- def testResetOffsetsExistingTopicAllGroups(): Unit = {
- val args = buildArgsForAllGroups("--topic", topic, "--to-offset", "50")
- produceMessages(topic, 100)
- for (group <- 1 to 3 map (group + _)) {
- val executor = addConsumerGroupExecutor(numConsumers = 1, topic = topic, group = group)
- awaitConsumerProgress(count = 100L, group = group)
- executor.shutdown()
- }
- resetAndAssertOffsets(args, expectedOffset = 50, dryRun = true)
- resetAndAssertOffsets(args ++ Array("--dry-run"), expectedOffset = 50, dryRun = true)
- resetAndAssertOffsets(args ++ Array("--execute"), expectedOffset = 50)
- }
-
- @Test
- def testResetOffsetsAllTopicsAllGroups(): Unit = {
- val args = buildArgsForAllGroups("--all-topics", "--to-offset", "50")
- val topics = 1 to 3 map (topic + _)
- val groups = 1 to 3 map (group + _)
- topics foreach (topic => produceMessages(topic, 100))
- for {
- topic <- topics
- group <- groups
- } {
- val executor = addConsumerGroupExecutor(numConsumers = 3, topic = topic, group = group)
- awaitConsumerProgress(topic = topic, count = 100L, group = group)
- executor.shutdown()
- }
- resetAndAssertOffsets(args, expectedOffset = 50, dryRun = true, topics = topics)
- resetAndAssertOffsets(args ++ Array("--dry-run"), expectedOffset = 50, dryRun = true, topics = topics)
- resetAndAssertOffsets(args ++ Array("--execute"), expectedOffset = 50, topics = topics)
- }
-
- @Test
- def testResetOffsetsToLocalDateTime(): Unit = {
- val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")
- val calendar = Calendar.getInstance()
- calendar.add(Calendar.DATE, -1)
-
- produceMessages(topic, 100)
-
- val executor = addConsumerGroupExecutor(numConsumers = 1, topic)
- awaitConsumerProgress(count = 100L)
- executor.shutdown()
-
- val args = buildArgsForGroup(group, "--all-topics", "--to-datetime", format.format(calendar.getTime), "--execute")
- resetAndAssertOffsets(args, expectedOffset = 0)
- }
-
- @Test
- def testResetOffsetsToZonedDateTime(): Unit = {
- val format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX")
-
- produceMessages(topic, 50)
- val checkpoint = new Date()
- produceMessages(topic, 50)
-
- val executor = addConsumerGroupExecutor(numConsumers = 1, topic)
- awaitConsumerProgress(count = 100L)
- executor.shutdown()
-
- val args = buildArgsForGroup(group, "--all-topics", "--to-datetime", format.format(checkpoint), "--execute")
- resetAndAssertOffsets(args, expectedOffset = 50)
- }
-
- @Test
- def testResetOffsetsByDuration(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--by-duration", "PT1M", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- resetAndAssertOffsets(args, expectedOffset = 0)
- }
-
- @Test
- def testResetOffsetsByDurationToEarliest(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--by-duration", "PT0.1S", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- resetAndAssertOffsets(args, expectedOffset = 100)
- }
-
- @Test
- def testResetOffsetsByDurationFallbackToLatestWhenNoRecords(): Unit = {
- val topic = "foo2"
- val args = buildArgsForGroup(group, "--topic", topic, "--by-duration", "PT1M", "--execute")
- createTopic(topic)
- resetAndAssertOffsets(args, expectedOffset = 0, topics = Seq("foo2"))
-
- adminZkClient.deleteTopic(topic)
- }
-
- @Test
- def testResetOffsetsToEarliest(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--to-earliest", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- resetAndAssertOffsets(args, expectedOffset = 0)
- }
-
- @Test
- def testResetOffsetsToLatest(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--to-latest", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- produceMessages(topic, 100)
- resetAndAssertOffsets(args, expectedOffset = 200)
- }
-
- @Test
- def testResetOffsetsToCurrentOffset(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--to-current", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- produceMessages(topic, 100)
- resetAndAssertOffsets(args, expectedOffset = 100)
- }
-
- @Test
- def testResetOffsetsToSpecificOffset(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--to-offset", "1", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- resetAndAssertOffsets(args, expectedOffset = 1)
- }
-
- @Test
- def testResetOffsetsShiftPlus(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "50", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- produceMessages(topic, 100)
- resetAndAssertOffsets(args, expectedOffset = 150)
- }
-
- @Test
- def testResetOffsetsShiftMinus(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "-50", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- produceMessages(topic, 100)
- resetAndAssertOffsets(args, expectedOffset = 50)
- }
-
- @Test
- def testResetOffsetsShiftByLowerThanEarliest(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "-150", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- produceMessages(topic, 100)
- resetAndAssertOffsets(args, expectedOffset = 0)
- }
-
- @Test
- def testResetOffsetsShiftByHigherThanLatest(): Unit = {
- val args = buildArgsForGroup(group, "--all-topics", "--shift-by", "150", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- produceMessages(topic, 100)
- resetAndAssertOffsets(args, expectedOffset = 200)
- }
-
- @Test
- def testResetOffsetsToEarliestOnOneTopic(): Unit = {
- val args = buildArgsForGroup(group, "--topic", topic, "--to-earliest", "--execute")
- produceConsumeAndShutdown(topic, group, totalMessages = 100)
- resetAndAssertOffsets(args, expectedOffset = 0)
- }
-
- @Test
- def testResetOffsetsToEarliestOnOneTopicAndPartition(): Unit = {
- val topic = "bar"
- createTopic(topic, 2, 1)
-
- val args = buildArgsForGroup(group, "--topic", s"$topic:1", "--to-earliest", "--execute")
- val consumerGroupCommand = getConsumerGroupService(args)
-
- produceConsumeAndShutdown(topic, group, totalMessages = 100, numConsumers = 2)
- val priorCommittedOffsets = committedOffsets(topic = topic)
-
- val tp0 = new TopicPartition(topic, 0)
- val tp1 = new TopicPartition(topic, 1)
- val expectedOffsets = Map(tp0 -> priorCommittedOffsets(tp0), tp1 -> 0L)
- resetAndAssertOffsetsCommitted(consumerGroupCommand, expectedOffsets, topic)
-
- adminZkClient.deleteTopic(topic)
- }
-
- @Test
- def testResetOffsetsToEarliestOnTopics(): Unit = {
- val topic1 = "topic1"
- val topic2 = "topic2"
- createTopic(topic1, 1, 1)
- createTopic(topic2, 1, 1)
-
- val args = buildArgsForGroup(group, "--topic", topic1, "--topic", topic2, "--to-earliest", "--execute")
- val consumerGroupCommand = getConsumerGroupService(args)
-
- produceConsumeAndShutdown(topic1, group, 100, 1)
- produceConsumeAndShutdown(topic2, group, 100, 1)
-
- val tp1 = new TopicPartition(topic1, 0)
- val tp2 = new TopicPartition(topic2, 0)
-
- val allResetOffsets = resetOffsets(consumerGroupCommand)(group).map { case (k, v) => k -> v.offset }
- assertEquals(Map(tp1 -> 0L, tp2 -> 0L), allResetOffsets)
- assertEquals(Map(tp1 -> 0L), committedOffsets(topic1))
- assertEquals(Map(tp2 -> 0L), committedOffsets(topic2))
-
- adminZkClient.deleteTopic(topic1)
- adminZkClient.deleteTopic(topic2)
- }
-
- @Test
- def testResetOffsetsToEarliestOnTopicsAndPartitions(): Unit = {
- val topic1 = "topic1"
- val topic2 = "topic2"
-
- createTopic(topic1, 2, 1)
- createTopic(topic2, 2, 1)
-
- val args = buildArgsForGroup(group, "--topic", s"$topic1:1", "--topic", s"$topic2:1", "--to-earliest", "--execute")
- val consumerGroupCommand = getConsumerGroupService(args)
-
- produceConsumeAndShutdown(topic1, group, 100, 2)
- produceConsumeAndShutdown(topic2, group, 100, 2)
-
- val priorCommittedOffsets1 = committedOffsets(topic1)
- val priorCommittedOffsets2 = committedOffsets(topic2)
-
- val tp1 = new TopicPartition(topic1, 1)
- val tp2 = new TopicPartition(topic2, 1)
- val allResetOffsets = resetOffsets(consumerGroupCommand)(group).map { case (k, v) => k -> v.offset }
- assertEquals(Map(tp1 -> 0, tp2 -> 0), allResetOffsets)
-
- assertEquals(priorCommittedOffsets1.toMap + (tp1 -> 0L), committedOffsets(topic1))
- assertEquals(priorCommittedOffsets2.toMap + (tp2 -> 0L), committedOffsets(topic2))
-
- adminZkClient.deleteTopic(topic1)
- adminZkClient.deleteTopic(topic2)
- }
-
- @Test
- // This one deals with old CSV export/import format for a single --group arg: "topic,partition,offset" to support old behavior
- def testResetOffsetsExportImportPlanSingleGroupArg(): Unit = {
- val topic = "bar"
- val tp0 = new TopicPartition(topic, 0)
- val tp1 = new TopicPartition(topic, 1)
- createTopic(topic, 2, 1)
-
- val cgcArgs = buildArgsForGroup(group, "--all-topics", "--to-offset", "2", "--export")
- val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
- produceConsumeAndShutdown(topic = topic, group = group, totalMessages = 100, numConsumers = 2)
-
- val file = TestUtils.tempFile("reset", ".csv")
-
- val exportedOffsets = consumerGroupCommand.resetOffsets()
- val bw = new BufferedWriter(new FileWriter(file))
- bw.write(consumerGroupCommand.exportOffsetsToCsv(exportedOffsets))
- bw.close()
- assertEquals(Map(tp0 -> 2L, tp1 -> 2L), exportedOffsets(group).map { case (k, v) => k -> v.offset })
-
- val cgcArgsExec = buildArgsForGroup(group, "--all-topics", "--from-file", file.getCanonicalPath, "--dry-run")
- val consumerGroupCommandExec = getConsumerGroupService(cgcArgsExec)
- val importedOffsets = consumerGroupCommandExec.resetOffsets()
- assertEquals(Map(tp0 -> 2L, tp1 -> 2L), importedOffsets(group).map { case (k, v) => k -> v.offset })
-
- adminZkClient.deleteTopic(topic)
- }
-
- @Test
- // This one deals with universal CSV export/import file format "group,topic,partition,offset",
- // supporting multiple --group args or --all-groups arg
- def testResetOffsetsExportImportPlan(): Unit = {
- val group1 = group + "1"
- val group2 = group + "2"
- val topic1 = "bar1"
- val topic2 = "bar2"
- val t1p0 = new TopicPartition(topic1, 0)
- val t1p1 = new TopicPartition(topic1, 1)
- val t2p0 = new TopicPartition(topic2, 0)
- val t2p1 = new TopicPartition(topic2, 1)
- createTopic(topic1, 2, 1)
- createTopic(topic2, 2, 1)
-
- val cgcArgs = buildArgsForGroups(Seq(group1, group2), "--all-topics", "--to-offset", "2", "--export")
- val consumerGroupCommand = getConsumerGroupService(cgcArgs)
-
- produceConsumeAndShutdown(topic = topic1, group = group1, totalMessages = 100)
- produceConsumeAndShutdown(topic = topic2, group = group2, totalMessages = 100)
-
- awaitConsumerGroupInactive(consumerGroupCommand, group1)
- awaitConsumerGroupInactive(consumerGroupCommand, group2)
-
- val file = TestUtils.tempFile("reset", ".csv")
-
- val exportedOffsets = consumerGroupCommand.resetOffsets()
- val bw = new BufferedWriter(new FileWriter(file))
- bw.write(consumerGroupCommand.exportOffsetsToCsv(exportedOffsets))
- bw.close()
- assertEquals(Map(t1p0 -> 2L, t1p1 -> 2L), exportedOffsets(group1).map { case (k, v) => k -> v.offset })
- assertEquals(Map(t2p0 -> 2L, t2p1 -> 2L), exportedOffsets(group2).map { case (k, v) => k -> v.offset })
-
- // Multiple --group's offset import
- val cgcArgsExec = buildArgsForGroups(Seq(group1, group2), "--all-topics", "--from-file", file.getCanonicalPath, "--dry-run")
- val consumerGroupCommandExec = getConsumerGroupService(cgcArgsExec)
- val importedOffsets = consumerGroupCommandExec.resetOffsets()
- assertEquals(Map(t1p0 -> 2L, t1p1 -> 2L), importedOffsets(group1).map { case (k, v) => k -> v.offset })
- assertEquals(Map(t2p0 -> 2L, t2p1 -> 2L), importedOffsets(group2).map { case (k, v) => k -> v.offset })
-
- // Single --group offset import using "group,topic,partition,offset" csv format
- val cgcArgsExec2 = buildArgsForGroup(group1, "--all-topics", "--from-file", file.getCanonicalPath, "--dry-run")
- val consumerGroupCommandExec2 = getConsumerGroupService(cgcArgsExec2)
- val importedOffsets2 = consumerGroupCommandExec2.resetOffsets()
- assertEquals(Map(t1p0 -> 2L, t1p1 -> 2L), importedOffsets2(group1).map { case (k, v) => k -> v.offset })
-
- adminZkClient.deleteTopic(topic)
- }
-
- @Test
- def testResetWithUnrecognizedNewConsumerOption(): Unit = {
- val cgcArgs = Array("--new-consumer", "--bootstrap-server", bootstrapServers(), "--reset-offsets", "--group", group, "--all-topics",
- "--to-offset", "2", "--export")
- assertThrows(classOf[OptionException], () => getConsumerGroupService(cgcArgs))
- }
-
- private def produceMessages(topic: String, numMessages: Int): Unit = {
- val records = (0 until numMessages).map(_ => new ProducerRecord[Array[Byte], Array[Byte]](topic,
- new Array[Byte](100 * 1000)))
- TestUtils.produceMessages(servers, records, acks = 1)
- }
-
- private def produceConsumeAndShutdown(topic: String, group: String, totalMessages: Int, numConsumers: Int = 1): Unit = {
- produceMessages(topic, totalMessages)
- val executor = addConsumerGroupExecutor(numConsumers = numConsumers, topic = topic, group = group)
- awaitConsumerProgress(topic, group, totalMessages)
- executor.shutdown()
- }
-
- private def awaitConsumerProgress(topic: String = topic,
- group: String = group,
- count: Long): Unit = {
- val consumer = createNoAutoCommitConsumer(group)
- try {
- val partitions = consumer.partitionsFor(topic).asScala.map { partitionInfo =>
- new TopicPartition(partitionInfo.topic, partitionInfo.partition)
- }.toSet
-
- TestUtils.waitUntilTrue(() => {
- val committed = consumer.committed(partitions.asJava).values.asScala
- val total = committed.foldLeft(0L) { case (currentSum, offsetAndMetadata) =>
- currentSum + Option(offsetAndMetadata).map(_.offset).getOrElse(0L)
- }
- total == count
- }, "Expected that consumer group has consumed all messages from topic/partition. " +
- s"Expected offset: $count. Actual offset: ${committedOffsets(topic, group).values.sum}")
-
- } finally {
- consumer.close()
- }
-
- }
-
- private def awaitConsumerGroupInactive(consumerGroupService: ConsumerGroupService, group: String): Unit = {
- TestUtils.waitUntilTrue(() => {
- val state = consumerGroupService.collectGroupState(group).state
- state == "Empty" || state == "Dead"
- }, s"Expected that consumer group is inactive. Actual state: ${consumerGroupService.collectGroupState(group).state}")
- }
-
- private def resetAndAssertOffsets(args: Array[String],
- expectedOffset: Long,
- dryRun: Boolean = false,
- topics: Seq[String] = Seq(topic)): Unit = {
- val consumerGroupCommand = getConsumerGroupService(args)
- val expectedOffsets = topics.map(topic => topic -> Map(new TopicPartition(topic, 0) -> expectedOffset)).toMap
- val resetOffsetsResultByGroup = resetOffsets(consumerGroupCommand)
-
- try {
- for {
- topic <- topics
- (group, partitionInfo) <- resetOffsetsResultByGroup
- } {
- val priorOffsets = committedOffsets(topic = topic, group = group)
- assertEquals(expectedOffsets(topic),
- partitionInfo.filter(partitionInfo => partitionInfo._1.topic() == topic).map { case (k, v) => k -> v.offset })
- assertEquals(if (dryRun) priorOffsets else expectedOffsets(topic), committedOffsets(topic = topic, group = group))
- }
- } finally {
- consumerGroupCommand.close()
- }
- }
-
- private def resetAndAssertOffsetsCommitted(consumerGroupService: ConsumerGroupService,
- expectedOffsets: Map[TopicPartition, Long],
- topic: String): Unit = {
- val allResetOffsets = resetOffsets(consumerGroupService)
- for {
- (group, offsetsInfo) <- allResetOffsets
- (tp, offsetMetadata) <- offsetsInfo
- } {
- assertEquals(offsetMetadata.offset(), expectedOffsets(tp))
- assertEquals(expectedOffsets, committedOffsets(topic, group))
- }
- }
-
- private def resetOffsets(consumerGroupService: ConsumerGroupService) = {
- consumerGroupService.resetOffsets()
- }
-}
diff --git a/tools/src/test/java/org/apache/kafka/tools/consumer/group/ConsumerGroupCommandTest.java b/tools/src/test/java/org/apache/kafka/tools/consumer/group/ConsumerGroupCommandTest.java
index bde3af37a1d..e3ee39f6b1d 100644
--- a/tools/src/test/java/org/apache/kafka/tools/consumer/group/ConsumerGroupCommandTest.java
+++ b/tools/src/test/java/org/apache/kafka/tools/consumer/group/ConsumerGroupCommandTest.java
@@ -87,10 +87,7 @@ public class ConsumerGroupCommandTest extends kafka.integration.KafkaServerTestH
0,
false
).foreach(props -> {
- if (isNewGroupCoordinatorEnabled()) {
- props.setProperty(KafkaConfig.NewGroupCoordinatorEnableProp(), "true");
- }
-
+ props.setProperty(KafkaConfig.NewGroupCoordinatorEnableProp(), isNewGroupCoordinatorEnabled() + "");
cfgs.add(KafkaConfig.fromProps(props));
return null;
});
diff --git a/tools/src/test/java/org/apache/kafka/tools/consumer/group/ResetConsumerGroupOffsetTest.java b/tools/src/test/java/org/apache/kafka/tools/consumer/group/ResetConsumerGroupOffsetTest.java
new file mode 100644
index 00000000000..dca26345223
--- /dev/null
+++ b/tools/src/test/java/org/apache/kafka/tools/consumer/group/ResetConsumerGroupOffsetTest.java
@@ -0,0 +1,589 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.tools.consumer.group;
+
+import joptsimple.OptionException;
+import kafka.admin.ConsumerGroupCommand;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.GroupProtocol;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.test.TestUtils;
+import org.junit.jupiter.api.Test;
+import scala.Option;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Properties;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Test cases by:
+ * - Non-existing consumer group
+ * - One for each scenario, with scope=all-topics
+ * - scope=one topic, scenario=to-earliest
+ * - scope=one topic+partitions, scenario=to-earliest
+ * - scope=topics, scenario=to-earliest
+ * - scope=topics+partitions, scenario=to-earliest
+ * - export/import
+ */
+public class ResetConsumerGroupOffsetTest extends ConsumerGroupCommandTest {
+ private String[] basicArgs() {
+ return new String[]{"--reset-offsets",
+ "--bootstrap-server", bootstrapServers(listenerName()),
+ "--timeout", Long.toString(DEFAULT_MAX_WAIT_MS)};
+ }
+
+ private String[] buildArgsForGroups(List<String> groups, String...args) {
+ List<String> res = new ArrayList<>(Arrays.asList(basicArgs()));
+ for (String group : groups) {
+ res.add("--group");
+ res.add(group);
+ }
+ res.addAll(Arrays.asList(args));
+ return res.toArray(new String[0]);
+ }
+
+ private String[] buildArgsForGroup(String group, String...args) {
+ return buildArgsForGroups(Collections.singletonList(group), args);
+ }
+
+ private String[] buildArgsForAllGroups(String...args) {
+ List<String> res = new ArrayList<>(Arrays.asList(basicArgs()));
+ res.add("--all-groups");
+ res.addAll(Arrays.asList(args));
+ return res.toArray(new String[0]);
+ }
+
+ @Test
+ public void testResetOffsetsNotExistingGroup() throws Exception {
+ String group = "missing.group";
+ String[] args = buildArgsForGroup(group, "--all-topics", "--to-current", "--execute");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommand = getConsumerGroupService(args);
+ // Make sure we got a coordinator
+ TestUtils.waitForCondition(
+ () -> Objects.equals(consumerGroupCommand.collectGroupState(group).coordinator().host(), "localhost"),
+ "Can't find a coordinator");
+ Option<scala.collection.Map<TopicPartition, OffsetAndMetadata>> resetOffsets = consumerGroupCommand.resetOffsets().get(group);
+ assertTrue(resetOffsets.isDefined() && resetOffsets.get().isEmpty());
+ assertTrue(committedOffsets(TOPIC, group).isEmpty());
+ }
+
+ @Test
+ public void testResetOffsetsExistingTopic() {
+ String group = "new.group";
+ String[] args = buildArgsForGroup(group, "--topic", TOPIC, "--to-offset", "50");
+ produceMessages(TOPIC, 100);
+ resetAndAssertOffsets(args, 50, true, Collections.singletonList(TOPIC));
+ resetAndAssertOffsets(addTo(args, "--dry-run"), 50, true, Collections.singletonList(TOPIC));
+ resetAndAssertOffsets(addTo(args, "--execute"), 50, false, Collections.singletonList(TOPIC));
+ }
+
+ @Test
+ public void testResetOffsetsExistingTopicSelectedGroups() throws Exception {
+ produceMessages(TOPIC, 100);
+ List<String> groups = IntStream.rangeClosed(1, 3).mapToObj(id -> GROUP + id).collect(Collectors.toList());
+ for (String group : groups) {
+ ConsumerGroupExecutor executor = addConsumerGroupExecutor(1, TOPIC, group, GroupProtocol.CLASSIC.name);
+ awaitConsumerProgress(TOPIC, group, 100L);
+ executor.shutdown();
+ }
+ String[] args = buildArgsForGroups(groups, "--topic", TOPIC, "--to-offset", "50");
+ resetAndAssertOffsets(args, 50, true, Collections.singletonList(TOPIC));
+ resetAndAssertOffsets(addTo(args, "--dry-run"), 50, true, Collections.singletonList(TOPIC));
+ resetAndAssertOffsets(addTo(args, "--execute"), 50, false, Collections.singletonList(TOPIC));
+ }
+
+ @Test
+ public void testResetOffsetsExistingTopicAllGroups() throws Exception {
+ String[] args = buildArgsForAllGroups("--topic", TOPIC, "--to-offset", "50");
+ produceMessages(TOPIC, 100);
+ for (int i = 1; i <= 3; i++) {
+ String group = GROUP + i;
+ ConsumerGroupExecutor executor = addConsumerGroupExecutor(1, TOPIC, group, GroupProtocol.CLASSIC.name);
+ awaitConsumerProgress(TOPIC, group, 100L);
+ executor.shutdown();
+ }
+ resetAndAssertOffsets(args, 50, true, Collections.singletonList(TOPIC));
+ resetAndAssertOffsets(addTo(args, "--dry-run"), 50, true, Collections.singletonList(TOPIC));
+ resetAndAssertOffsets(addTo(args, "--execute"), 50, false, Collections.singletonList(TOPIC));
+ }
+
+ @Test
+ public void testResetOffsetsAllTopicsAllGroups() throws Exception {
+ String[] args = buildArgsForAllGroups("--all-topics", "--to-offset", "50");
+ List<String> topics = IntStream.rangeClosed(1, 3).mapToObj(i -> TOPIC + i).collect(Collectors.toList());
+ List<String> groups = IntStream.rangeClosed(1, 3).mapToObj(i -> GROUP + i).collect(Collectors.toList());
+ topics.forEach(topic -> produceMessages(topic, 100));
+
+ for (String topic : topics) {
+ for (String group : groups) {
+ ConsumerGroupExecutor executor = addConsumerGroupExecutor(3, topic, group, GroupProtocol.CLASSIC.name);
+ awaitConsumerProgress(topic, group, 100);
+ executor.shutdown();
+ }
+ }
+ resetAndAssertOffsets(args, 50, true, topics);
+ resetAndAssertOffsets(addTo(args, "--dry-run"), 50, true, topics);
+ resetAndAssertOffsets(addTo(args, "--execute"), 50, false, topics);
+ }
+
+ @Test
+ public void testResetOffsetsToLocalDateTime() throws Exception {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");
+ Calendar calendar = Calendar.getInstance();
+ calendar.add(Calendar.DATE, -1);
+
+ produceMessages(TOPIC, 100);
+
+ ConsumerGroupExecutor executor = addConsumerGroupExecutor(1, TOPIC, GROUP, GroupProtocol.CLASSIC.name);
+ awaitConsumerProgress(TOPIC, GROUP, 100L);
+ executor.shutdown();
+
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--to-datetime", format.format(calendar.getTime()), "--execute");
+ resetAndAssertOffsets(args, 0);
+ }
+
+ @Test
+ public void testResetOffsetsToZonedDateTime() throws Exception {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX");
+
+ produceMessages(TOPIC, 50);
+ Date checkpoint = new Date();
+ produceMessages(TOPIC, 50);
+
+ ConsumerGroupExecutor executor = addConsumerGroupExecutor(1, TOPIC, GROUP, GroupProtocol.CLASSIC.name);
+ awaitConsumerProgress(TOPIC, GROUP, 100L);
+ executor.shutdown();
+
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--to-datetime", format.format(checkpoint), "--execute");
+ resetAndAssertOffsets(args, 50);
+ }
+
+ @Test
+ public void testResetOffsetsByDuration() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--by-duration", "PT1M", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ resetAndAssertOffsets(args, 0);
+ }
+
+ @Test
+ public void testResetOffsetsByDurationToEarliest() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--by-duration", "PT0.1S", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ resetAndAssertOffsets(args, 100);
+ }
+
+ @Test
+ public void testResetOffsetsByDurationFallbackToLatestWhenNoRecords() throws Exception {
+ String topic = "foo2";
+ String[] args = buildArgsForGroup(GROUP, "--topic", topic, "--by-duration", "PT1M", "--execute");
+ createTopic(topic, 1, 1, new Properties(), listenerName(), new Properties());
+ resetAndAssertOffsets(args, 0, false, Collections.singletonList("foo2"));
+
+ adminZkClient().deleteTopic(topic);
+ }
+
+ @Test
+ public void testResetOffsetsToEarliest() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--to-earliest", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ resetAndAssertOffsets(args, 0);
+ }
+
+ @Test
+ public void testResetOffsetsToLatest() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--to-latest", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ produceMessages(TOPIC, 100);
+ resetAndAssertOffsets(args, 200);
+ }
+
+ @Test
+ public void testResetOffsetsToCurrentOffset() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--to-current", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ produceMessages(TOPIC, 100);
+ resetAndAssertOffsets(args, 100);
+ }
+
+ @Test
+ public void testResetOffsetsToSpecificOffset() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--to-offset", "1", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ resetAndAssertOffsets(args, 1);
+ }
+
+ @Test
+ public void testResetOffsetsShiftPlus() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--shift-by", "50", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ produceMessages(TOPIC, 100);
+ resetAndAssertOffsets(args, 150);
+ }
+
+ @Test
+ public void testResetOffsetsShiftMinus() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--shift-by", "-50", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ produceMessages(TOPIC, 100);
+ resetAndAssertOffsets(args, 50);
+ }
+
+ @Test
+ public void testResetOffsetsShiftByLowerThanEarliest() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--shift-by", "-150", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ produceMessages(TOPIC, 100);
+ resetAndAssertOffsets(args, 0);
+ }
+
+ @Test
+ public void testResetOffsetsShiftByHigherThanLatest() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--all-topics", "--shift-by", "150", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ produceMessages(TOPIC, 100);
+ resetAndAssertOffsets(args, 200);
+ }
+
+ @Test
+ public void testResetOffsetsToEarliestOnOneTopic() throws Exception {
+ String[] args = buildArgsForGroup(GROUP, "--topic", TOPIC, "--to-earliest", "--execute");
+ produceConsumeAndShutdown(TOPIC, GROUP, 100, 1);
+ resetAndAssertOffsets(args, 0);
+ }
+
+ @Test
+ public void testResetOffsetsToEarliestOnOneTopicAndPartition() throws Exception {
+ String topic = "bar";
+ createTopic(topic, 2, 1, new Properties(), listenerName(), new Properties());
+
+ String[] args = buildArgsForGroup(GROUP, "--topic", topic + ":1", "--to-earliest", "--execute");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommand = getConsumerGroupService(args);
+
+ produceConsumeAndShutdown(topic, GROUP, 100, 2);
+ Map<TopicPartition, Long> priorCommittedOffsets = committedOffsets(topic, GROUP);
+
+ TopicPartition tp0 = new TopicPartition(topic, 0);
+ TopicPartition tp1 = new TopicPartition(topic, 1);
+ Map<TopicPartition, Long> expectedOffsets = new HashMap<>();
+ expectedOffsets.put(tp0, priorCommittedOffsets.get(tp0));
+ expectedOffsets.put(tp1, 0L);
+ resetAndAssertOffsetsCommitted(consumerGroupCommand, expectedOffsets, topic);
+
+ adminZkClient().deleteTopic(topic);
+ }
+
+ @Test
+ public void testResetOffsetsToEarliestOnTopics() throws Exception {
+ String topic1 = "topic1";
+ String topic2 = "topic2";
+ createTopic(topic1, 1, 1, new Properties(), listenerName(), new Properties());
+ createTopic(topic2, 1, 1, new Properties(), listenerName(), new Properties());
+
+ String[] args = buildArgsForGroup(GROUP, "--topic", topic1, "--topic", topic2, "--to-earliest", "--execute");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommand = getConsumerGroupService(args);
+
+ produceConsumeAndShutdown(topic1, GROUP, 100, 1);
+ produceConsumeAndShutdown(topic2, GROUP, 100, 1);
+
+ TopicPartition tp1 = new TopicPartition(topic1, 0);
+ TopicPartition tp2 = new TopicPartition(topic2, 0);
+
+ Map<TopicPartition, Long> allResetOffsets = toOffsetMap(consumerGroupCommand.resetOffsets().get(GROUP));
+ Map<TopicPartition, Long> expMap = new HashMap<>();
+ expMap.put(tp1, 0L);
+ expMap.put(tp2, 0L);
+ assertEquals(expMap, allResetOffsets);
+ assertEquals(Collections.singletonMap(tp1, 0L), committedOffsets(topic1, GROUP));
+ assertEquals(Collections.singletonMap(tp2, 0L), committedOffsets(topic2, GROUP));
+
+ adminZkClient().deleteTopic(topic1);
+ adminZkClient().deleteTopic(topic2);
+ }
+
+ @Test
+ public void testResetOffsetsToEarliestOnTopicsAndPartitions() throws Exception {
+ String topic1 = "topic1";
+ String topic2 = "topic2";
+
+ createTopic(topic1, 2, 1, new Properties(), listenerName(), new Properties());
+ createTopic(topic2, 2, 1, new Properties(), listenerName(), new Properties());
+
+ String[] args = buildArgsForGroup(GROUP, "--topic", topic1 + ":1", "--topic", topic2 + ":1", "--to-earliest", "--execute");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommand = getConsumerGroupService(args);
+
+ produceConsumeAndShutdown(topic1, GROUP, 100, 2);
+ produceConsumeAndShutdown(topic2, GROUP, 100, 2);
+
+ Map<TopicPartition, Long> priorCommittedOffsets1 = committedOffsets(topic1, GROUP);
+ Map<TopicPartition, Long> priorCommittedOffsets2 = committedOffsets(topic2, GROUP);
+
+ TopicPartition tp1 = new TopicPartition(topic1, 1);
+ TopicPartition tp2 = new TopicPartition(topic2, 1);
+ Map<TopicPartition, Long> allResetOffsets = toOffsetMap(consumerGroupCommand.resetOffsets().get(GROUP));
+ Map<TopicPartition, Long> expMap = new HashMap<>();
+ expMap.put(tp1, 0L);
+ expMap.put(tp2, 0L);
+ assertEquals(expMap, allResetOffsets);
+ priorCommittedOffsets1.put(tp1, 0L);
+ assertEquals(priorCommittedOffsets1, committedOffsets(topic1, GROUP));
+ priorCommittedOffsets2.put(tp2, 0L);
+ assertEquals(priorCommittedOffsets2, committedOffsets(topic2, GROUP));
+
+ adminZkClient().deleteTopic(topic1);
+ adminZkClient().deleteTopic(topic2);
+ }
+
+ @Test
+ // This one deals with old CSV export/import format for a single --group arg: "topic,partition,offset" to support old behavior
+ public void testResetOffsetsExportImportPlanSingleGroupArg() throws Exception {
+ String topic = "bar";
+ TopicPartition tp0 = new TopicPartition(topic, 0);
+ TopicPartition tp1 = new TopicPartition(topic, 1);
+ createTopic(topic, 2, 1, new Properties(), listenerName(), new Properties());
+
+ String[] cgcArgs = buildArgsForGroup(GROUP, "--all-topics", "--to-offset", "2", "--export");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommand = getConsumerGroupService(cgcArgs);
+
+ produceConsumeAndShutdown(topic, GROUP, 100, 2);
+
+ File file = TestUtils.tempFile("reset", ".csv");
+
+ scala.collection.Map<String, scala.collection.Map<TopicPartition, OffsetAndMetadata>> exportedOffsets = consumerGroupCommand.resetOffsets();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(file));
+ bw.write(consumerGroupCommand.exportOffsetsToCsv(exportedOffsets));
+ bw.close();
+
+ Map<TopicPartition, Long> exp1 = new HashMap<>();
+ exp1.put(tp0, 2L);
+ exp1.put(tp1, 2L);
+ assertEquals(exp1, toOffsetMap(exportedOffsets.get(GROUP)));
+
+ String[] cgcArgsExec = buildArgsForGroup(GROUP, "--all-topics", "--from-file", file.getCanonicalPath(), "--dry-run");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommandExec = getConsumerGroupService(cgcArgsExec);
+ scala.collection.Map<String, scala.collection.Map<TopicPartition, OffsetAndMetadata>> importedOffsets = consumerGroupCommandExec.resetOffsets();
+ assertEquals(exp1, toOffsetMap(importedOffsets.get(GROUP)));
+
+ adminZkClient().deleteTopic(topic);
+ }
+
+ @Test
+ // This one deals with universal CSV export/import file format "group,topic,partition,offset",
+ // supporting multiple --group args or --all-groups arg
+ public void testResetOffsetsExportImportPlan() throws Exception {
+ String group1 = GROUP + "1";
+ String group2 = GROUP + "2";
+ String topic1 = "bar1";
+ String topic2 = "bar2";
+ TopicPartition t1p0 = new TopicPartition(topic1, 0);
+ TopicPartition t1p1 = new TopicPartition(topic1, 1);
+ TopicPartition t2p0 = new TopicPartition(topic2, 0);
+ TopicPartition t2p1 = new TopicPartition(topic2, 1);
+ createTopic(topic1, 2, 1, new Properties(), listenerName(), new Properties());
+ createTopic(topic2, 2, 1, new Properties(), listenerName(), new Properties());
+
+ String[] cgcArgs = buildArgsForGroups(Arrays.asList(group1, group2), "--all-topics", "--to-offset", "2", "--export");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommand = getConsumerGroupService(cgcArgs);
+
+ produceConsumeAndShutdown(topic1, group1, 100, 1);
+ produceConsumeAndShutdown(topic2, group2, 100, 1);
+
+ awaitConsumerGroupInactive(consumerGroupCommand, group1);
+ awaitConsumerGroupInactive(consumerGroupCommand, group2);
+
+ File file = TestUtils.tempFile("reset", ".csv");
+
+ scala.collection.Map<String, scala.collection.Map<TopicPartition, OffsetAndMetadata>> exportedOffsets = consumerGroupCommand.resetOffsets();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(file));
+ bw.write(consumerGroupCommand.exportOffsetsToCsv(exportedOffsets));
+ bw.close();
+ Map<TopicPartition, Long> exp1 = new HashMap<>();
+ exp1.put(t1p0, 2L);
+ exp1.put(t1p1, 2L);
+ Map<TopicPartition, Long> exp2 = new HashMap<>();
+ exp2.put(t2p0, 2L);
+ exp2.put(t2p1, 2L);
+
+ assertEquals(exp1, toOffsetMap(exportedOffsets.get(group1)));
+ assertEquals(exp2, toOffsetMap(exportedOffsets.get(group2)));
+
+ // Multiple --group's offset import
+ String[] cgcArgsExec = buildArgsForGroups(Arrays.asList(group1, group2), "--all-topics", "--from-file", file.getCanonicalPath(), "--dry-run");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommandExec = getConsumerGroupService(cgcArgsExec);
+ scala.collection.Map<String, scala.collection.Map<TopicPartition, OffsetAndMetadata>> importedOffsets = consumerGroupCommandExec.resetOffsets();
+ assertEquals(exp1, toOffsetMap(importedOffsets.get(group1)));
+ assertEquals(exp2, toOffsetMap(importedOffsets.get(group2)));
+
+ // Single --group offset import using "group,topic,partition,offset" csv format
+ String[] cgcArgsExec2 = buildArgsForGroup(group1, "--all-topics", "--from-file", file.getCanonicalPath(), "--dry-run");
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommandExec2 = getConsumerGroupService(cgcArgsExec2);
+ scala.collection.Map<String, scala.collection.Map<TopicPartition, OffsetAndMetadata>> importedOffsets2 = consumerGroupCommandExec2.resetOffsets();
+ assertEquals(exp1, toOffsetMap(importedOffsets2.get(group1)));
+
+ adminZkClient().deleteTopic(TOPIC);
+ }
+
+ @Test
+ public void testResetWithUnrecognizedNewConsumerOption() {
+ String[] cgcArgs = new String[]{"--new-consumer", "--bootstrap-server", bootstrapServers(listenerName()), "--reset-offsets",
+ "--group", GROUP, "--all-topics", "--to-offset", "2", "--export"};
+ assertThrows(OptionException.class, () -> getConsumerGroupService(cgcArgs));
+ }
+
+ private void produceMessages(String topic, int numMessages) {
+ List<ProducerRecord<byte[], byte[]>> records = IntStream.range(0, numMessages)
+ .mapToObj(i -> new ProducerRecord<byte[], byte[]>(topic, new byte[100 * 1000]))
+ .collect(Collectors.toList());
+ kafka.utils.TestUtils.produceMessages(servers(), seq(records), 1);
+ }
+
+ private void produceConsumeAndShutdown(String topic, String group, int totalMessages, int numConsumers) throws Exception {
+ produceMessages(topic, totalMessages);
+ ConsumerGroupExecutor executor = addConsumerGroupExecutor(numConsumers, topic, group, GroupProtocol.CLASSIC.name);
+ awaitConsumerProgress(topic, group, totalMessages);
+ executor.shutdown();
+ }
+
+ private void awaitConsumerProgress(String topic,
+ String group,
+ long count) throws Exception {
+ try (Consumer<String, String> consumer = createNoAutoCommitConsumer(group)) {
+ Set<TopicPartition> partitions = consumer.partitionsFor(topic).stream()
+ .map(partitionInfo -> new TopicPartition(partitionInfo.topic(), partitionInfo.partition()))
+ .collect(Collectors.toSet());
+
+ TestUtils.waitForCondition(() -> {
+ Collection<OffsetAndMetadata> committed = consumer.committed(partitions).values();
+ long total = committed.stream()
+ .mapToLong(offsetAndMetadata -> Optional.ofNullable(offsetAndMetadata).map(OffsetAndMetadata::offset).orElse(0L))
+ .sum();
+
+ return total == count;
+ }, "Expected that consumer group has consumed all messages from topic/partition. " +
+ "Expected offset: " + count + ". Actual offset: " + committedOffsets(topic, group).values().stream().mapToLong(Long::longValue).sum());
+ }
+ }
+
+ private void awaitConsumerGroupInactive(ConsumerGroupCommand.ConsumerGroupService consumerGroupService, String group) throws Exception {
+ TestUtils.waitForCondition(() -> {
+ String state = consumerGroupService.collectGroupState(group).state();
+ return Objects.equals(state, "Empty") || Objects.equals(state, "Dead");
+ }, "Expected that consumer group is inactive. Actual state: " + consumerGroupService.collectGroupState(group).state());
+ }
+
+ private void resetAndAssertOffsets(String[] args,
+ long expectedOffset) {
+ resetAndAssertOffsets(args, expectedOffset, false, Collections.singletonList(TOPIC));
+ }
+
+ private void resetAndAssertOffsets(String[] args,
+ long expectedOffset,
+ boolean dryRun,
+ List<String> topics) {
+ ConsumerGroupCommand.ConsumerGroupService consumerGroupCommand = getConsumerGroupService(args);
+ Map<String, Map<TopicPartition, Long>> expectedOffsets = topics.stream().collect(Collectors.toMap(
+ Function.identity(),
+ topic -> Collections.singletonMap(new TopicPartition(topic, 0), expectedOffset)));
+ scala.collection.Map<String, scala.collection.Map<TopicPartition, OffsetAndMetadata>> resetOffsetsResultByGroup = consumerGroupCommand.resetOffsets();
+
+ try {
+ for (final String topic : topics) {
+ resetOffsetsResultByGroup.foreach(entry -> {
+ String group = entry._1;
+ scala.collection.Map<TopicPartition, OffsetAndMetadata> partitionInfo = entry._2;
+ Map<TopicPartition, Long> priorOffsets = committedOffsets(topic, group);
+ Map<TopicPartition, Long> offsets = new HashMap<>();
+ partitionInfo.foreach(partitionInfoEntry -> {
+ TopicPartition tp = partitionInfoEntry._1;
+ OffsetAndMetadata offsetAndMetadata = partitionInfoEntry._2;
+ if (Objects.equals(tp.topic(), topic))
+ offsets.put(tp, offsetAndMetadata.offset());
+ return null;
+ });
+ assertEquals(expectedOffsets.get(topic), offsets);
+ assertEquals(dryRun ? priorOffsets : expectedOffsets.get(topic), committedOffsets(topic, group));
+ return null;
+ });
+ }
+ } finally {
+ consumerGroupCommand.close();
+ }
+ }
+
+ private void resetAndAssertOffsetsCommitted(ConsumerGroupCommand.ConsumerGroupService consumerGroupService,
+ Map<TopicPartition, Long> expectedOffsets,
+ String topic) {
+ scala.collection.Map<String, scala.collection.Map<TopicPartition, OffsetAndMetadata>> allResetOffsets = consumerGroupService.resetOffsets();
+
+ allResetOffsets.foreach(entry -> {
+ String group = entry._1;
+ scala.collection.Map<TopicPartition, OffsetAndMetadata> offsetsInfo = entry._2;
+ offsetsInfo.foreach(offsetInfoEntry -> {
+ TopicPartition tp = offsetInfoEntry._1;
+ OffsetAndMetadata offsetMetadata = offsetInfoEntry._2;
+ assertEquals(offsetMetadata.offset(), expectedOffsets.get(tp));
+ assertEquals(expectedOffsets, committedOffsets(topic, group));
+ return null;
+ });
+ return null;
+ });
+ }
+
+ Map<TopicPartition, Long> toOffsetMap(Option<scala.collection.Map<TopicPartition, OffsetAndMetadata>> map) {
+ assertTrue(map.isDefined());
+ Map<TopicPartition, Long> res = new HashMap<>();
+ map.foreach(m -> {
+ m.foreach(entry -> {
+ TopicPartition tp = entry._1;
+ OffsetAndMetadata offsetAndMetadata = entry._2;
+ res.put(tp, offsetAndMetadata.offset());
+ return null;
+ });
+ return null;
+ });
+ return res;
+ }
+
+ private String[] addTo(String[] args, String...extra) {
+ List<String> res = new ArrayList<>(Arrays.asList(args));
+ res.addAll(Arrays.asList(extra));
+ return res.toArray(new String[0]);
+ }
+}