You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by jo...@apache.org on 2024/12/19 00:27:35 UTC
(kafka) 01/01: Merge branch 'trunk' of github.com:apache/kafka into kafka-17705
This is an automated email from the ASF dual-hosted git repository.
jolshan pushed a commit to branch kafka-17705
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit 2496a60a89ae35c302a71ce75020d58e5ed3d22f
Merge: 92dfdf60287 f7f3cffb483
Author: Justine <jo...@confluent.io>
AuthorDate: Wed Dec 18 16:21:30 2024 -0800
Merge branch 'trunk' of github.com:apache/kafka into kafka-17705
.github/scripts/develocity_reports.py | 1074 ++++++++++++++++++++
.github/scripts/requirements.txt | 2 +
.github/workflows/README.md | 10 +-
.github/workflows/build.yml | 12 +-
.github/workflows/ci.yml | 4 +-
.github/workflows/docker_scan.yml | 2 +-
...r-reviewed-trigger.yml => generate-reports.yml} | 36 +-
.github/workflows/pr-labeled.yml | 5 +-
.github/workflows/pr-reviewed.yml | 50 +-
LICENSE-binary | 9 +-
README.md | 17 +-
bin/connect-distributed.sh | 8 +-
bin/connect-mirror-maker.sh | 8 +-
bin/connect-standalone.sh | 8 +-
bin/kafka-acls.sh | 2 +-
bin/kafka-server-start.sh | 8 +-
bin/windows/connect-distributed.bat | 9 +-
bin/windows/connect-standalone.bat | 9 +-
bin/windows/kafka-acls.bat | 2 +-
bin/windows/kafka-server-start.bat | 9 +-
build.gradle | 217 ++--
checkstyle/import-control-core.xml | 19 +-
checkstyle/import-control.xml | 42 +-
checkstyle/suppressions.xml | 4 +-
.../apache/kafka/clients/CommonClientConfigs.java | 3 +-
.../clients/admin/DescribeClusterOptions.java | 15 +
.../kafka/clients/admin/KafkaAdminClient.java | 13 +-
.../apache/kafka/clients/admin/TopicListing.java | 14 -
.../kafka/clients/consumer/ConsumerConfig.java | 4 +-
.../internals/AbstractMembershipManager.java | 10 +-
.../consumer/internals/AsyncKafkaConsumer.java | 115 ++-
.../internals/AutoOffsetResetStrategy.java | 17 +
.../consumer/internals/ConsumerNetworkThread.java | 22 +-
.../clients/consumer/internals/ConsumerUtils.java | 1 +
.../internals/CoordinatorRequestManager.java | 28 +-
.../consumer/internals/NetworkClientDelegate.java | 31 +-
.../consumer/internals/ShareConsumerImpl.java | 26 +-
.../consumer/internals/SubscriptionState.java | 4 +-
.../internals/events/ApplicationEvent.java | 16 +-
.../internals/events/ApplicationEventHandler.java | 13 +-
.../consumer/internals/events/BackgroundEvent.java | 16 +-
.../internals/events/BackgroundEventHandler.java | 30 +-
.../internals/events/CompletableEventReaper.java | 24 +-
.../internals/metrics/AsyncConsumerMetrics.java | 262 +++++
.../kafka/clients/producer/KafkaProducer.java | 1 +
.../producer/internals/TransactionManager.java | 26 +-
.../main/java/org/apache/kafka/common/Node.java | 26 +-
.../errors/NotLeaderForPartitionException.java | 44 -
.../errors/NotLeaderOrFollowerException.java | 3 +-
.../StreamsInvalidTopologyEpochException.java} | 14 +-
.../StreamsInvalidTopologyException.java} | 14 +-
.../StreamsTopologyFencedException.java} | 12 +-
.../org/apache/kafka/common/protocol/ApiKeys.java | 5 +-
.../org/apache/kafka/common/protocol/Errors.java | 8 +-
.../kafka/common/requests/AbstractRequest.java | 4 +
.../kafka/common/requests/AbstractResponse.java | 4 +
.../common/requests/DescribeClusterResponse.java | 2 +-
.../requests/StreamsGroupDescribeRequest.java | 99 ++
...onse.java => StreamsGroupDescribeResponse.java} | 50 +-
.../requests/StreamsGroupHeartbeatRequest.java | 88 ++
.../requests/StreamsGroupHeartbeatResponse.java | 103 ++
.../apache/kafka/common/requests/VoteRequest.java | 8 +-
.../internals/ClientTelemetryReporter.java | 4 +
.../org/apache/kafka/common/utils/ThreadUtils.java | 31 +-
.../java/org/apache/kafka/common/utils/Utils.java | 34 +-
.../apache/kafka/server/authorizer/Authorizer.java | 6 +-
.../common/message/DescribeClusterRequest.json | 7 +-
.../common/message/DescribeClusterResponse.json | 7 +-
.../common/message/SaslHandshakeRequest.json | 1 -
.../message/StreamsGroupDescribeRequest.json | 16 +-
.../message/StreamsGroupDescribeResponse.json | 167 +++
.../message/StreamsGroupHeartbeatRequest.json | 132 +++
.../message/StreamsGroupHeartbeatResponse.json | 117 +++
.../main/resources/common/message/VoteRequest.json | 23 +-
.../resources/common/message/VoteResponse.json | 3 +-
.../kafka/clients/admin/KafkaAdminClientTest.java | 17 +
.../kafka/clients/consumer/KafkaConsumerTest.java | 2 +-
.../internals/AbstractCoordinatorTest.java | 2 +
.../internals/ApplicationEventHandlerTest.java | 73 ++
.../consumer/internals/AsyncKafkaConsumerTest.java | 33 +-
.../internals/BackgroundEventHandlerTest.java | 65 ++
.../internals/ConsumerMembershipManagerTest.java | 34 +-
.../internals/ConsumerNetworkClientTest.java | 2 +
.../internals/ConsumerNetworkThreadTest.java | 99 +-
.../internals/CoordinatorRequestManagerTest.java | 80 ++
.../internals/FetchRequestManagerTest.java | 3 +-
.../internals/NetworkClientDelegateTest.java | 53 +-
.../internals/ShareConsumeRequestManagerTest.java | 15 +-
.../consumer/internals/ShareConsumerImplTest.java | 2 +-
.../events/CompletableEventReaperTest.java | 16 +-
.../metrics/AsyncConsumerMetricsTest.java | 237 +++++
.../kafka/clients/producer/KafkaProducerTest.java | 2 +-
.../producer/internals/TransactionManagerTest.java | 7 +-
.../common/message/NullableStructMessageTest.java | 26 +
.../common/message/SimpleExampleMessageTest.java | 2 +
.../kafka/common/requests/RequestResponseTest.java | 72 +-
.../internals/ClientTelemetryReporterTest.java | 3 +-
.../kafka/common/utils/LogCaptureAppender.java | 128 ++-
clients/src/test/resources/log4j2.yaml | 38 +
committer-tools/kafka-merge-pr.py | 2 +-
config/connect-log4j.properties | 39 -
config/connect-log4j2.yaml | 44 +
config/log4j.properties | 93 --
config/log4j2.yaml | 158 +++
.../kafka/connect/rest/ConnectRestExtension.java | 2 +-
.../connect/rest/ConnectRestExtensionContext.java | 9 +-
.../apache/kafka/connect/source/SourceTask.java | 28 +-
.../extension/BasicAuthSecurityRestExtension.java | 2 +-
.../basic/auth/extension/JaasBasicAuthFilter.java | 13 +-
.../BasicAuthSecurityRestExtensionTest.java | 3 +-
.../auth/extension/JaasBasicAuthFilterTest.java | 11 +-
connect/file/src/test/resources/log4j.properties | 28 -
.../file/src/test/resources/log4j2.yaml | 29 +-
.../connect/mirror/MirrorCheckpointConnector.java | 14 +-
.../rest/resources/InternalMirrorResource.java | 10 +-
.../connect/mirror/MirrorSourceConnectorTest.java | 2 +-
.../DedicatedMirrorIntegrationTest.java | 8 +-
connect/mirror/src/test/resources/log4j.properties | 33 -
.../mirror/src/test/resources/log4j2.yaml | 34 +-
.../kafka/connect/runtime/AbstractHerder.java | 411 +++++---
.../kafka/connect/runtime/CachedConnectors.java | 79 ++
.../kafka/connect/runtime/ConnectorConfig.java | 238 ++++-
.../org/apache/kafka/connect/runtime/Herder.java | 4 +
.../org/apache/kafka/connect/runtime/Loggers.java | 101 +-
.../kafka/connect/runtime/SinkConnectorConfig.java | 28 +-
.../connect/runtime/SourceConnectorConfig.java | 16 +-
.../org/apache/kafka/connect/runtime/Worker.java | 4 +
.../apache/kafka/connect/runtime/WorkerConfig.java | 20 +
.../runtime/distributed/DistributedHerder.java | 5 +-
.../runtime/isolation/DelegatingClassLoader.java | 25 +-
.../connect/runtime/isolation/PluginDesc.java | 3 +-
.../connect/runtime/isolation/PluginUtils.java | 11 +-
.../kafka/connect/runtime/isolation/Plugins.java | 20 +-
.../runtime/isolation/PluginsRecommenders.java | 198 ++++
.../runtime/rest/ConnectRestConfigurable.java | 4 +-
.../rest/ConnectRestExtensionContextImpl.java | 2 +-
.../connect/runtime/rest/HerderRequestHandler.java | 7 +-
.../runtime/rest/InternalRequestSignature.java | 11 +-
.../kafka/connect/runtime/rest/RestClient.java | 27 +-
.../kafka/connect/runtime/rest/RestServer.java | 33 +-
.../runtime/rest/errors/BadRequestException.java | 2 +-
.../rest/errors/ConnectExceptionMapper.java | 14 +-
.../runtime/rest/errors/ConnectRestException.java | 2 +-
.../rest/resources/ConnectorPluginsResource.java | 42 +-
.../runtime/rest/resources/ConnectorsResource.java | 43 +-
.../rest/resources/InternalClusterResource.java | 23 +-
.../rest/resources/InternalConnectResource.java | 4 +-
.../runtime/rest/resources/LoggingResource.java | 25 +-
.../runtime/rest/resources/RootResource.java | 13 +-
.../connect/integration/BlockingConnectorTest.java | 15 +-
.../integration/ConnectWorkerIntegrationTest.java | 4 +-
.../ConnectorRestartApiIntegrationTest.java | 2 +-
.../integration/InternalTopicsIntegrationTest.java | 2 +-
.../integration/OffsetsApiIntegrationTest.java | 4 +-
.../integration/RestExtensionIntegrationTest.java | 8 +-
.../SessionedProtocolIntegrationTest.java | 4 +-
.../StandaloneWorkerIntegrationTest.java | 2 +-
.../kafka/connect/runtime/AbstractHerderTest.java | 66 +-
.../apache/kafka/connect/runtime/LoggersTest.java | 137 +--
.../connect/runtime/SampleSourceConnector.java | 2 +-
.../runtime/SourceTaskOffsetCommitterTest.java | 2 +-
.../connect/runtime/WorkerSourceTaskTest.java | 2 +-
.../runtime/distributed/DistributedHerderTest.java | 10 +-
.../runtime/isolation/SynchronizationTest.java | 4 +-
.../runtime/rest/ConnectRestServerTest.java | 3 +-
.../runtime/rest/InternalRequestSignatureTest.java | 25 +-
.../kafka/connect/runtime/rest/RestClientTest.java | 15 +-
.../resources/ConnectorPluginsResourceTest.java | 6 +-
.../rest/resources/ConnectorsResourceTest.java | 12 +-
.../resources/InternalConnectResourceTest.java | 5 +-
.../rest/resources/LoggingResourceTest.java | 4 +-
.../runtime/rest/resources/RootResourceTest.java | 2 +-
.../connect/runtime/rest/util/SSLUtilsTest.java | 69 +-
.../runtime/standalone/StandaloneHerderTest.java | 30 +-
.../connect/util/clusters/ConnectAssertions.java | 2 +-
.../connect/util/clusters/EmbeddedConnect.java | 19 +-
.../util/clusters/EmbeddedConnectStandalone.java | 2 +-
.../runtime/src/test/resources/log4j.properties | 37 -
connect/runtime/src/test/resources/log4j2.yaml | 48 +
.../common/runtime/CoordinatorRuntime.java | 33 +-
.../common/runtime/PartitionWriter.java | 11 +
.../common/runtime/CoordinatorRuntimeTest.java | 102 +-
.../common/runtime/InMemoryPartitionWriter.java | 8 +
.../java/kafka/log/remote/RemoteLogManager.java | 90 +-
.../java/kafka/server/share/ShareFetchUtils.java | 14 +
.../java/kafka/server/share/SharePartition.java | 13 +-
core/src/main/scala/kafka/Kafka.scala | 9 +-
core/src/main/scala/kafka/admin/AclCommand.scala | 510 ----------
.../scala/kafka/controller/KafkaController.scala | 6 +-
.../group/CoordinatorPartitionWriter.scala | 21 +
.../kafka/coordinator/group/GroupCoordinator.scala | 129 +--
.../group/GroupCoordinatorAdapter.scala | 16 +-
.../coordinator/group/GroupMetadataManager.scala | 16 +-
.../main/scala/kafka/log/OffsetResultHolder.scala | 12 +-
core/src/main/scala/kafka/raft/RaftManager.scala | 2 -
.../scala/kafka/server/DynamicBrokerConfig.scala | 42 +-
core/src/main/scala/kafka/server/KafkaApis.scala | 65 +-
core/src/main/scala/kafka/server/KafkaConfig.scala | 43 +-
core/src/main/scala/kafka/server/KafkaServer.scala | 146 +--
.../kafka/server/ListOffsetsPartitionStatus.scala | 2 +-
.../main/scala/kafka/server/MetadataCache.scala | 7 +-
.../main/scala/kafka/server/ReplicaManager.scala | 37 +-
.../kafka/server/metadata/KRaftMetadataCache.scala | 4 +
.../kafka/server/metadata/ZkMetadataCache.scala | 10 +-
.../main/scala/kafka/tools/DumpLogSegments.scala | 6 +-
.../main/scala/kafka/utils/Log4jController.scala | 127 ++-
core/src/main/scala/kafka/utils/ToolsUtils.scala | 35 -
core/src/main/scala/kafka/zk/KafkaZkClient.scala | 250 +----
core/src/main/scala/kafka/zk/ZkData.scala | 39 -
.../AsyncKafkaConsumerIntegrationTest.java | 73 --
.../clients/consumer/ConsumerIntegrationTest.java | 188 ++++
.../log/remote/RemoteLogOffsetReaderTest.java | 2 +-
.../kafka/server/share/SharePartitionTest.java | 118 ++-
.../java/kafka/test/api/ShareConsumerTest.java | 485 ++++-----
.../src/test/resources/log4j2.yaml | 32 +-
.../kafka/api/AdminClientRebootstrapTest.scala | 2 +-
.../kafka/api/BaseProducerSendTest.scala | 2 +-
.../kafka/api/ConsumerRebootstrapTest.scala | 9 +-
.../kafka/api/CustomQuotaCallbackTest.scala | 13 +-
.../kafka/api/EndToEndAuthorizationTest.scala | 2 +-
.../kafka/api/EndToEndClusterIdTest.scala | 2 +-
.../kafka/api/IntegrationTestHarness.scala | 26 +-
.../kafka/api/PlaintextAdminIntegrationTest.scala | 84 +-
.../kafka/api/ProducerCompressionTest.scala | 2 +-
.../kafka/api/ProducerFailureHandlingTest.scala | 2 +-
.../kafka/api/ProducerIdExpirationTest.scala | 2 +-
.../kafka/api/ProducerRebootstrapTest.scala | 4 +-
.../kafka/api/ProducerSendWhileDeletionTest.scala | 2 +-
.../integration/kafka/api/RebootstrapTest.scala | 8 +-
.../kafka/api/SslAdminIntegrationTest.scala | 23 +-
.../kafka/api/TransactionsBounceTest.scala | 2 +-
.../kafka/api/TransactionsExpirationTest.scala | 2 +-
.../api/TransactionsWithMaxInFlightOneTest.scala | 2 +-
.../server/DelayedRemoteListOffsetsTest.scala | 2 +-
.../server/FetchFromFollowerIntegrationTest.scala | 2 +-
.../kafka/server/QuorumTestHarness.scala | 144 +--
.../src/test/scala/kafka/utils/TestInfoUtils.scala | 15 +-
core/src/test/scala/other/kafka.log4j.properties | 22 -
.../scala/unit/kafka/admin/AddPartitionsTest.scala | 5 +-
.../group/CoordinatorPartitionWriterTest.scala | 82 +-
.../group/GroupCoordinatorAdapterTest.scala | 25 +-
.../kafka/integration/KafkaServerTestHarness.scala | 207 +---
.../MetricsDuringTopicCreationDeletionTest.scala | 10 +-
.../unit/kafka/integration/MinIsrConfigTest.scala | 2 +-
.../integration/UncleanLeaderElectionTest.scala | 19 +-
.../scala/unit/kafka/metrics/MetricsTest.scala | 12 +-
.../unit/kafka/network/SocketServerTest.scala | 9 +-
.../scala/unit/kafka/server/BaseRequestTest.scala | 31 +-
.../server/BrokerRegistrationRequestTest.scala | 33 +-
...leteTopicsRequestWithDeletionDisabledTest.scala | 2 +-
.../kafka/server/DynamicBrokerConfigTest.scala | 75 ++
.../unit/kafka/server/EdgeCaseRequestTest.scala | 2 +-
.../scala/unit/kafka/server/KafkaApisTest.scala | 138 ++-
.../scala/unit/kafka/server/KafkaConfigTest.scala | 43 +-
.../kafka/server/KafkaMetricsReporterTest.scala | 2 +-
.../unit/kafka/server/LeaveGroupRequestTest.scala | 49 +-
.../unit/kafka/server/LogDirFailureTest.scala | 32 +-
.../scala/unit/kafka/server/LogRecoveryTest.scala | 2 +-
.../unit/kafka/server/MetadataCacheTest.scala | 69 --
.../unit/kafka/server/ReplicaManagerTest.scala | 133 ++-
.../unit/kafka/server/ReplicationQuotasTest.scala | 8 +-
.../scala/unit/kafka/server/RequestQuotaTest.scala | 6 +
.../server/epoch/LeaderEpochIntegrationTest.scala | 8 +-
.../test/scala/unit/kafka/utils/TestUtils.scala | 78 +-
docs/js/templateData.js | 6 +-
docs/ops.html | 120 +--
docs/streams/developer-guide/config-streams.html | 104 +-
docs/upgrade.html | 28 +
.../apache/kafka/message/MessageDataGenerator.java | 19 +-
gradle.properties | 2 +-
gradle/dependencies.gradle | 37 +-
gradle/spotbugs-exclude.xml | 13 +
.../assignor/ConsumerGroupPartitionAssignor.java | 6 -
.../group/api/assignor/GroupAssignment.java | 3 -
.../coordinator/group/api/assignor/GroupSpec.java | 2 -
.../group/api/assignor/MemberAssignment.java | 2 -
.../group/api/assignor/MemberSubscription.java | 2 -
.../group/api/assignor/PartitionAssignor.java | 6 -
.../api/assignor/SubscribedTopicDescriber.java | 5 -
.../group/api/assignor/SubscriptionType.java | 3 -
.../kafka/coordinator/group/GroupConfig.java | 32 +-
.../kafka/coordinator/group/GroupCoordinator.java | 9 +-
.../group/GroupCoordinatorRecordSerde.java | 3 +
.../coordinator/group/GroupCoordinatorService.java | 2 +-
.../coordinator/group/GroupMetadataManager.java | 42 +-
.../group/ShareGroupAutoOffsetResetStrategy.java | 159 +++
.../ConsumerGroupCurrentMemberAssignmentKey.json | 1 -
.../ConsumerGroupCurrentMemberAssignmentValue.json | 1 -
.../message/ConsumerGroupMemberMetadataKey.json | 1 -
.../message/ConsumerGroupMemberMetadataValue.json | 1 -
.../common/message/ConsumerGroupMetadataKey.json | 1 -
.../common/message/ConsumerGroupMetadataValue.json | 1 -
.../message/ConsumerGroupPartitionMetadataKey.json | 1 -
.../ConsumerGroupPartitionMetadataValue.json | 1 -
.../message/ConsumerGroupRegularExpressionKey.json | 1 -
.../ConsumerGroupRegularExpressionValue.json | 1 -
.../ConsumerGroupTargetAssignmentMemberKey.json | 1 -
.../ConsumerGroupTargetAssignmentMemberValue.json | 1 -
.../ConsumerGroupTargetAssignmentMetadataKey.json | 1 -
...ConsumerGroupTargetAssignmentMetadataValue.json | 1 -
.../kafka/coordinator/group/GroupConfigTest.java | 16 +
.../group/GroupMetadataManagerTest.java | 51 +-
.../ShareGroupAutoOffsetResetStrategyTest.java | 112 ++
.../src/test/resources/log4j2.yaml | 26 +-
.../jmh/fetcher/ReplicaFetcherThreadBenchmark.java | 2 +-
.../controller/ActivationRecordsGenerator.java | 32 +-
.../kafka/controller/FeatureControlManager.java | 32 -
.../apache/kafka/controller/QuorumController.java | 6 +-
.../apache/kafka/controller/QuorumFeatures.java | 22 -
.../controller/ReplicationControlManager.java | 2 +-
.../ControllerMetadataMetricsPublisher.java | 4 -
.../java/org/apache/kafka/image/FeaturesDelta.java | 21 +-
.../java/org/apache/kafka/image/FeaturesImage.java | 31 +-
.../java/org/apache/kafka/image/MetadataDelta.java | 9 +-
.../apache/kafka/image/node/FeaturesImageNode.java | 2 -
.../apache/kafka/metadata/BrokerRegistration.java | 2 +-
.../migration/ZkMigrationLeadershipState.java | 202 ----
.../kafka/metadata/migration/ZkMigrationState.java | 108 --
.../common/metadata/ZkMigrationRecord.json | 4 +-
.../controller/ActivationRecordsGeneratorTest.java | 120 +--
.../controller/FeatureControlManagerTest.java | 38 -
.../kafka/controller/QuorumControllerTest.java | 38 +-
.../kafka/controller/QuorumFeaturesTest.java | 56 -
.../org/apache/kafka/image/FeaturesImageTest.java | 14 +-
.../org/apache/kafka/image/ImageDowngradeTest.java | 2 -
.../kafka/image/publisher/SnapshotEmitterTest.java | 2 +-
.../kafka/metadata/BrokerRegistrationTest.java | 2 +-
.../src/test/resources/log4j2.yaml | 30 +-
raft/bin/test-kraft-server-start.sh | 1 +
.../log4j.properties => config/kraft-log4j2.yaml} | 33 +-
.../java/org/apache/kafka/raft/CandidateState.java | 19 +-
.../java/org/apache/kafka/raft/EpochState.java | 11 +-
.../java/org/apache/kafka/raft/FollowerState.java | 23 +-
.../org/apache/kafka/raft/KafkaRaftClient.java | 117 ++-
.../java/org/apache/kafka/raft/LeaderState.java | 7 +-
.../java/org/apache/kafka/raft/QuorumState.java | 10 +-
.../main/java/org/apache/kafka/raft/RaftUtil.java | 29 +-
.../java/org/apache/kafka/raft/ResignedState.java | 14 +-
.../org/apache/kafka/raft/UnattachedState.java | 33 +-
.../org/apache/kafka/raft/CandidateStateTest.java | 13 +-
.../org/apache/kafka/raft/FollowerStateTest.java | 24 +-
.../kafka/raft/KafkaRaftClientPreVoteTest.java | 576 +++++++++++
.../org/apache/kafka/raft/KafkaRaftClientTest.java | 148 ++-
.../org/apache/kafka/raft/LeaderStateTest.java | 16 +-
.../apache/kafka/raft/RaftClientTestContext.java | 47 +-
.../java/org/apache/kafka/raft/RaftUtilTest.java | 44 +-
.../org/apache/kafka/raft/ResignedStateTest.java | 19 +-
.../org/apache/kafka/raft/UnattachedStateTest.java | 42 +-
.../kafka/raft/UnattachedStateWithVoteTest.java | 66 +-
.../resources/{log4j.properties => log4j2.yaml} | 32 +-
release/templates.py | 5 +-
.../src/test/resources/log4j2.yaml | 28 +-
.../apache/kafka/network/RequestConvertToJson.java | 16 +
.../apache/kafka/server/config/KRaftConfigs.java | 14 +-
.../server/config/ShareCoordinatorConfig.java | 14 +-
.../share/ShareCoordinatorOffsetsManager.java | 122 +++
.../coordinator/share/ShareCoordinatorService.java | 94 +-
.../coordinator/share/ShareCoordinatorShard.java | 33 +-
.../share/ShareCoordinatorOffsetsManagerTest.java | 209 ++++
.../share/ShareCoordinatorServiceTest.java | 505 ++++++++-
.../share/ShareCoordinatorShardTest.java | 23 +-
...igTest.java => ShareCoordinatorTestConfig.java} | 3 +-
.../src/test/resources/log4j2.yaml | 25 +-
.../log/remote/storage/RemoteLogManagerConfig.java | 28 +-
.../internals/log/AsyncOffsetReadFutureHolder.java | 22 +-
.../internals/log/RemoteStorageThreadPool.java | 30 +-
.../remote/storage/RemoteLogManagerConfigTest.java | 6 +-
storage/src/test/resources/log4j.properties | 28 -
storage/src/test/resources/log4j2.yaml | 57 ++
.../FineGrainedAutoResetIntegrationTest.java | 154 ++-
.../integration/utils/EmbeddedKafkaCluster.java | 7 +-
.../integration/utils/IntegrationTestUtils.java | 5 +-
.../src/test/resources/log4j.properties | 36 -
.../src/test/resources/log4j2.yaml | 65 ++
streams/quickstart/java/pom.xml | 2 +-
.../src/main/resources/archetype-resources/pom.xml | 2 +-
.../src/main/resources/log4j.properties | 19 -
.../src/main/resources/log4j2.yaml | 26 +-
streams/quickstart/pom.xml | 2 +-
.../org/apache/kafka/streams/AutoOffsetReset.java | 103 ++
.../org/apache/kafka/streams/StreamsBuilder.java | 4 +-
.../org/apache/kafka/streams/StreamsConfig.java | 44 +-
.../java/org/apache/kafka/streams/Topology.java | 251 ++++-
.../streams/internals/AutoOffsetResetInternal.java | 45 +
.../org/apache/kafka/streams/kstream/Consumed.java | 123 ++-
.../org/apache/kafka/streams/kstream/KStream.java | 203 ----
.../kstream/internals/ConsumedInternal.java | 20 +-
.../kstream/internals/InternalStreamsBuilder.java | 4 +-
.../streams/kstream/internals/KStreamImpl.java | 32 +-
.../streams/kstream/internals/KStreamImplJoin.java | 33 +-
.../kstream/internals/KStreamJoinWindow.java | 19 +-
.../kstream/internals/KStreamKStreamJoin.java | 37 +-
.../internals/KStreamKStreamJoinLeftSide.java | 13 +-
.../internals/KStreamKStreamJoinRightSide.java | 13 +-
.../kstream/internals/KStreamKStreamSelfJoin.java | 27 +-
.../streams/kstream/internals/KTableFilter.java | 18 +-
.../streams/kstream/internals/KTableImpl.java | 97 +-
.../internals/KTableKTableAbstractJoin.java | 8 +
.../kstream/internals/KTableKTableJoinMerger.java | 21 +-
.../streams/kstream/internals/KTableMapValues.java | 18 +-
.../ForeignTableJoinProcessorSupplier.java | 22 +-
.../SubscriptionReceiveProcessorSupplier.java | 24 +-
.../internals/graph/BaseJoinProcessorNode.java | 1 -
.../internals/graph/ForeignTableJoinNode.java | 8 +-
.../internals/graph/KTableKTableJoinNode.java | 54 +-
.../internals/graph/ProcessorParameters.java | 26 +-
.../kstream/internals/graph/StreamSourceNode.java | 2 +-
.../internals/graph/StreamStreamJoinNode.java | 131 +--
.../kstream/internals/graph/StreamToTableNode.java | 29 +-
.../kstream/internals/graph/TableFilterNode.java | 6 +-
.../internals/graph/TableProcessorNode.java | 21 +-
.../kstream/internals/graph/TableSuppressNode.java | 6 +-
.../suppress/KTableSuppressProcessorSupplier.java | 20 +-
.../internals/InternalTopologyBuilder.java | 76 +-
.../streams/processor/internals/SerdeGetter.java | 39 +-
.../streams/processor/internals/StreamThread.java | 146 ++-
.../streams/processor/internals/TaskManager.java | 12 +
.../processor/internals/TopologyMetadata.java | 7 +-
.../internals/metrics/ProcessorNodeMetrics.java | 5 +-
.../internals/metrics/StreamsMetricsImpl.java | 1 +
.../RocksDBTimeOrderedKeyValueBuffer.java | 4 +-
.../state/internals/StoreSerdeInitializer.java | 15 -
.../internals/metrics/RocksDBMetricsRecorder.java | 2 +-
.../apache/kafka/streams/AutoOffsetResetTest.java | 88 ++
.../apache/kafka/streams/StreamsBuilderTest.java | 486 ++++++++-
.../apache/kafka/streams/StreamsConfigTest.java | 2 +-
.../org/apache/kafka/streams/TopologyTest.java | 4 +-
.../internals/InternalStreamsBuilderTest.java | 71 +-
.../streams/kstream/internals/KStreamImplTest.java | 107 +-
.../kstream/internals/KStreamKStreamJoinTest.java | 53 +-
.../ForeignTableJoinProcessorSupplierTests.java | 9 +-
.../SubscriptionReceiveProcessorSupplierTest.java | 6 +-
.../KTableSuppressProcessorMetricsTest.java | 10 +-
.../suppress/KTableSuppressProcessorTest.java | 10 +-
.../internals/InternalTopicManagerTest.java | 2 +-
.../internals/InternalTopologyBuilderTest.java | 36 +-
.../processor/internals/PartitionGroupTest.java | 2 +-
.../processor/internals/RecordCollectorTest.java | 5 +-
.../internals/StoreChangelogReaderTest.java | 2 +-
.../processor/internals/StreamThreadTest.java | 5 +-
.../internals/StreamsPartitionAssignorTest.java | 5 +-
.../processor/internals/TaskManagerTest.java | 2 +-
.../RocksDBTimeOrderedKeyValueBufferTest.java | 6 +-
.../state/internals/StoreSerdeInitializerTest.java | 16 +-
.../internals/TimeOrderedKeyValueBufferTest.java | 86 +-
.../metrics/RocksDBBlockCacheMetricsTest.java | 27 +-
.../metrics/RocksDBMetricsRecorderTest.java | 16 +
.../test/MockInternalNewProcessorContext.java | 25 +-
.../kafka/test/MockInternalProcessorContext.java | 224 ----
streams/src/test/resources/log4j.properties | 36 -
streams/src/test/resources/log4j2.yaml | 65 ++
.../kafka/streams/scala/kstream/Consumed.scala | 36 +-
.../kafka/streams/scala/kstream/KStream.scala | 44 -
.../src/test/resources/log4j.properties | 34 -
.../streams-scala/src/test/resources/log4j2.yaml | 40 +
.../kafka/streams/scala/kstream/ConsumedTest.scala | 14 +-
.../test-utils/src/test/resources/log4j.properties | 21 -
.../test-utils/src/test/resources/log4j2.yaml | 28 +-
.../kafka/common/test/KafkaClusterTestKit.java | 7 +-
.../org/apache/kafka/common/test/TestUtils.java | 33 +-
.../src/main/resources/log4j2.yaml | 29 +-
.../kafka/common/test/api/ClusterInstance.java | 12 +-
.../common/test/api/ClusterTestExtensionsTest.java | 17 +-
tests/README.md | 4 +
tests/docker/Dockerfile | 4 +-
tests/kafkatest/__init__.py | 2 +-
tests/kafkatest/services/connect.py | 15 +-
tests/kafkatest/services/console_consumer.py | 14 +-
tests/kafkatest/services/kafka/kafka.py | 8 +-
.../kafkatest/services/kafka/templates/log4j2.yaml | 283 ++++++
tests/kafkatest/services/kafka/util.py | 17 +
.../services/performance/consumer_performance.py | 9 +-
.../services/performance/end_to_end_latency.py | 10 +-
.../services/performance/producer_performance.py | 9 +-
.../performance/templates/tools_log4j2.yaml | 24 +-
tests/kafkatest/services/streams.py | 39 +-
.../services/templates/connect_log4j2.yaml | 32 +-
.../kafkatest/services/templates/tools_log4j2.yaml | 39 +
.../services/transactional_message_copier.py | 10 +-
.../services/trogdor/templates/log4j2.yaml | 35 +-
tests/kafkatest/services/trogdor/trogdor.py | 20 +-
tests/kafkatest/services/verifiable_client.py | 2 +-
tests/kafkatest/services/verifiable_consumer.py | 26 +-
tests/kafkatest/services/verifiable_producer.py | 9 +-
.../client/consumer_protocol_migration_test.py | 2 +-
tests/kafkatest/tests/client/consumer_test.py | 28 +-
tests/kafkatest/tests/client/pluggable_test.py | 2 +-
tests/kafkatest/tests/connect/connect_rest_test.py | 6 +-
.../tests/streams/streams_relational_smoke_test.py | 14 +-
.../tests/streams/templates/log4j2_template.yaml | 30 +-
tests/kafkatest/tests/verifiable_consumer_test.py | 11 +-
tests/kafkatest/version.py | 9 +-
.../java/org/apache/kafka/tools/AclCommand.java | 605 +++++++++++
.../java/org/apache/kafka/tools/ClusterTool.java | 62 +-
.../org/apache/kafka/tools/StreamsResetter.java | 9 +
.../org/apache/kafka/tools/VerifiableConsumer.java | 11 +-
.../tools/consumer/group/ConsumerGroupCommand.java | 244 +++--
.../tools/consumer/group/GroupInformation.java | 16 +-
.../consumer/group/MemberAssignmentState.java | 24 +-
.../consumer/group/PartitionAssignmentState.java | 19 +-
.../kafka/tools/AbstractResetIntegrationTest.java | 2 +
.../org/apache/kafka/tools}/AclCommandTest.java | 324 +++---
.../kafka/tools/BrokerApiVersionsCommandTest.java | 2 +-
.../org/apache/kafka/tools/ClusterToolTest.java | 50 +
.../apache/kafka/tools/ResetIntegrationTest.java | 54 +
.../group/ConsumerGroupCommandTestUtils.java | 12 +-
.../consumer/group/ConsumerGroupServiceTest.java | 49 +-
.../consumer/group/DeleteConsumerGroupsTest.java | 2 +-
.../consumer/group/DescribeConsumerGroupTest.java | 300 +++++-
.../tools/other/ReplicationQuotasTestRig.java | 4 +-
tools/src/test/resources/log4j.properties | 22 -
.../src/test/resources/log4j2.yaml | 33 +-
.../apache/kafka/trogdor/agent/AgentClient.java | 2 +-
.../kafka/trogdor/agent/AgentRestResource.java | 24 +-
.../trogdor/coordinator/CoordinatorClient.java | 4 +-
.../coordinator/CoordinatorRestResource.java | 30 +-
.../apache/kafka/trogdor/rest/JsonRestServer.java | 16 +-
.../kafka/trogdor/rest/RestExceptionMapper.java | 6 +-
.../kafka/trogdor/coordinator/CoordinatorTest.java | 2 +-
.../trogdor/rest/RestExceptionMapperTest.java | 4 +-
trogdor/src/test/resources/log4j.properties | 22 -
.../src/test/resources/log4j2.yaml | 33 +-
vagrant/base.sh | 4 +-
523 files changed, 14460 insertions(+), 7243 deletions(-)
diff --cc clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
index 54241b093c3,ab99b3d3e46..ad73fa6ba37
--- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
+++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java
@@@ -445,8 -458,10 +458,11 @@@ public class TransactionManager
ApiVersions.FinalizedFeaturesInfo info = apiVersions.getFinalizedFeaturesInfo();
latestFinalizedFeaturesEpoch = info.finalizedFeaturesEpoch;
Short transactionVersion = info.finalizedFeatures.get("transaction.version");
+ boolean wasTransactionV2Enabled = isTransactionV2Enabled;
isTransactionV2Enabled = transactionVersion != null && transactionVersion >= 2;
+ log.debug("Updating isTV2 enabled to {} with FinalizedFeaturesEpoch {}", isTransactionV2Enabled, latestFinalizedFeaturesEpoch);
+ if (!onInitiatialization && !wasTransactionV2Enabled && isTransactionV2Enabled)
+ clientSideEpochBumpRequired = true;
}
public boolean isTransactionV2Enabled() {
diff --cc tests/kafkatest/services/kafka/kafka.py
index ea3c724e037,b713aacb04e..ba1b6937867
--- a/tests/kafkatest/services/kafka/kafka.py
+++ b/tests/kafkatest/services/kafka/kafka.py
@@@ -33,8 -33,7 +33,8 @@@ from kafkatest.services.security.listen
from kafkatest.services.security.security_config import SecurityConfig
from kafkatest.version import DEV_BRANCH
from kafkatest.version import KafkaVersion
+from kafkatest.version import get_version
- from kafkatest.services.kafka.util import fix_opts_for_new_jvm
+ from kafkatest.services.kafka.util import fix_opts_for_new_jvm, get_log4j_config_param, get_log4j_config
class KafkaListener:
diff --cc tests/kafkatest/version.py
index 2c04f3511f5,8278e71b051..6eeeedd4452
--- a/tests/kafkatest/version.py
+++ b/tests/kafkatest/version.py
@@@ -110,9 -107,8 +110,9 @@@ def get_version(node=None)
return DEV_BRANCH
DEV_BRANCH = KafkaVersion("dev")
- DEV_VERSION = KafkaVersion("4.0.0-SNAPSHOT")
+ DEV_VERSION = KafkaVersion("4.1.0-SNAPSHOT")
+LATEST_STABLE_TRANSACTION_VERSION = 2
# This should match the LATEST_PRODUCTION version defined in MetadataVersion.java
LATEST_STABLE_METADATA_VERSION = "4.0-IV0"