You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by vv...@apache.org on 2020/09/09 17:35:39 UTC
[kafka] branch kip-478-part-3 updated (3b81208 -> c376878)
This is an automated email from the ASF dual-hosted git repository.
vvcephei pushed a change to branch kip-478-part-3
in repository https://gitbox.apache.org/repos/asf/kafka.git.
from 3b81208 drop defaulted method
add d2c978c MINOR: fix JavaDoc (#9217)
add b6ba674 KAFKA-10384: Separate converters from generated messages (#9194)
add 9da32b6 KAFKA-9924: Add RocksDB metric num-entries-active-mem-table (#9177)
add 85b6545 KAFKA-5636: SlidingWindows (KIP-450) (#9039)
add d0111d3 KAFKA-10020: Create a new version of a scala Serdes without name clash (KIP-616) (#8955)
add 241e144 KAFKA-10444: Configure PR builds via Jenkinsfile (#9238)
add 59206b8 https://issues.apache.org/jira/browse/KAFKA-10456 (#9240)
add 6e8a01e KAFKA-10098: Remove unnecessary escaping in regular expression. (#8798)
add 55c1ab6 KAFKA-10272: Add IBM i support to "stop" scripts (#9023)
add fd02c8f MINOR: Include call name in TimeoutException (#8050)
add 4662ed4 MINOR: Fix build scala 2.12 build after KAFKA-10020 (#9245)
add 4f06d9e KAFKA-9929: Support backward iterator on WindowStore (#9138)
add 24b03a6 MINOR: Fix message count for sliding windows test (#9248)
add c04000c KAFKA-9924: Add remaining property-based RocksDB metrics as described in KIP-607 (#9232)
add 1e31354 KAFKA-10355: Throw error when source topic was deleted (#9191)
add 9c8501f MINOR: Record all poll invocations (#9234)
add 23cade8 KAFKA-10314: KafkaStorageException on reassignment when offline log directories exist (#9122)
add c6d135e KAFKA-10133: MM2 readme update on config (#9215)
add e8524cc KAFKA-10259: KIP-554 Broker-side SCRAM Config API (#9032)
add d7f3505 MINOR: Update scala default version in readme (#9260)
add d252185 KAFKA-10432; LeaderEpochCache is incorrectly recovered for leader epoch 0 (#9219)
add 4ea709d KAFKA-10403: Replace Scala collection by Java collection in Log4jController (#9182)
add 8e9cf0e MINOR: mirror integration tests should not call System.exit (#9200)
add c376878 Merge remote-tracking branch 'apache/trunk' into kip-478-part-3
No new revisions were added by this update.
Summary of changes:
Jenkinsfile | 169 +++++
README.md | 3 +-
bin/kafka-server-stop.sh | 5 +-
bin/zookeeper-server-stop.sh | 5 +-
build.gradle | 33 +-
checkstyle/import-control.xml | 2 +
checkstyle/suppressions.xml | 6 +-
.../java/org/apache/kafka/clients/admin/Admin.java | 92 +++
....java => AlterUserScramCredentialsOptions.java} | 6 +-
...t.java => AlterUserScramCredentialsResult.java} | 28 +-
...va => DescribeUserScramCredentialsOptions.java} | 6 +-
.../admin/DescribeUserScramCredentialsResult.java | 150 ++++
.../kafka/clients/admin/KafkaAdminClient.java | 190 ++++-
.../kafka/clients/admin/ScramCredentialInfo.java | 78 +++
.../apache/kafka/clients/admin/ScramMechanism.java | 81 +++
...ons.java => UserScramCredentialAlteration.java} | 28 +-
...cords.java => UserScramCredentialDeletion.java} | 29 +-
.../admin/UserScramCredentialUpsertion.java | 100 +++
.../admin/UserScramCredentialsDescription.java | 82 +++
.../common/errors/DuplicateResourceException.java | 77 +++
.../common/errors/ResourceNotFoundException.java | 76 ++
...n.java => UnacceptableCredentialException.java} | 23 +-
.../org/apache/kafka/common/protocol/ApiKeys.java | 10 +-
.../org/apache/kafka/common/protocol/Errors.java | 8 +-
.../org/apache/kafka/common/protocol/Message.java | 42 --
.../kafka/common/requests/AbstractRequest.java | 4 +
.../kafka/common/requests/AbstractResponse.java | 4 +
.../requests/AlterUserScramCredentialsRequest.java | 102 +++
...java => AlterUserScramCredentialsResponse.java} | 24 +-
.../DescribeUserScramCredentialsRequest.java | 87 +++
...a => DescribeUserScramCredentialsResponse.java} | 24 +-
.../security/scram/internals/ScramFormatter.java | 30 +-
.../security/scram/internals/ScramSaslClient.java | 4 +-
.../security/scram/internals/ScramSaslServer.java | 2 +-
.../message/AlterUserScramCredentialsRequest.json | 44 ++
...json => AlterUserScramCredentialsResponse.json} | 23 +-
...on => DescribeUserScramCredentialsRequest.json} | 18 +-
...n => DescribeUserScramCredentialsResponse.json} | 40 +-
.../DescribeUserScramCredentialsResultTest.java | 118 ++++
.../kafka/clients/admin/KafkaAdminClientTest.java | 154 +++++
.../kafka/clients/admin/MockAdminClient.java | 10 +
.../apache/kafka/common/message/MessageTest.java | 21 +-
.../kafka/common/protocol/ProtoUtilsTest.java | 18 +-
.../authenticator/SaslAuthenticatorTest.java | 2 +-
.../scram/internals/ScramFormatterTest.java | 12 +-
.../scram/internals/ScramMessagesTest.java | 4 +-
connect/mirror/README.md | 35 +-
.../mirror/MirrorConnectorsIntegrationTest.java | 14 +-
.../src/main/scala/kafka/admin/ConfigCommand.scala | 163 +++--
.../scala/kafka/controller/KafkaController.scala | 9 +-
core/src/main/scala/kafka/log/LogSegment.scala | 2 +-
.../src/main/scala/kafka/server/AdminManager.scala | 253 ++++++-
core/src/main/scala/kafka/server/KafkaApis.scala | 35 +-
.../main/scala/kafka/tools/ConsoleProducer.scala | 2 +-
.../main/scala/kafka/utils/Log4jController.scala | 5 +-
.../kafka/api/CustomQuotaCallbackTest.scala | 20 +-
.../DelegationTokenEndToEndAuthorizationTest.scala | 73 +-
.../kafka/api/EndToEndAuthorizationTest.scala | 20 +-
.../SaslClientsWithInvalidCredentialsTest.scala | 7 +-
.../SaslScramSslEndToEndAuthorizationTest.scala | 6 +-
.../scala/integration/kafka/api/SaslSetup.scala | 43 ++
.../server/DynamicBrokerReconfigurationTest.scala | 14 +-
.../kafka/tools/MirrorMakerIntegrationTest.scala | 23 +-
core/src/test/scala/kafka/utils/LoggingTest.scala | 8 +
.../scala/unit/kafka/admin/ConfigCommandTest.scala | 125 +++-
.../admin/UserScramCredentialsCommandTest.scala | 137 ++++
.../controller/ControllerIntegrationTest.scala | 43 +-
.../kafka/integration/KafkaServerTestHarness.scala | 9 +
.../test/scala/unit/kafka/log/LogSegmentTest.scala | 56 +-
.../AlterUserScramCredentialsRequestTest.scala | 433 ++++++++++++
.../kafka/server/ClientQuotasRequestTest.scala | 13 +-
.../DescribeUserScramCredentialsRequestTest.scala | 149 ++++
.../unit/kafka/server/LogDirFailureTest.scala | 38 +-
.../scala/unit/kafka/server/RequestQuotaTest.scala | 6 +
.../scala/unit/kafka/utils/JaasTestUtils.scala | 47 +-
.../test/scala/unit/kafka/utils/TestUtils.scala | 33 +-
docs/streams/developer-guide/dsl-api.html | 12 +-
.../java/org/apache/kafka/message/FieldSpec.java | 340 +++++++++
.../java/org/apache/kafka/message/FieldType.java | 65 ++
.../kafka/message/JsonConverterGenerator.java | 412 +++++++++++
...ssGenerator.java => MessageClassGenerator.java} | 21 +-
.../apache/kafka/message/MessageDataGenerator.java | 770 +++------------------
.../org/apache/kafka/message/MessageGenerator.java | 91 ++-
.../java/org/apache/kafka/message/MessageSpec.java | 2 +-
.../org/apache/kafka/message/SchemaGenerator.java | 2 +-
.../org/apache/kafka/message/StructRegistry.java | 74 +-
.../errors/MissingSourceTopicException.java | 9 +-
.../streams/errors/StreamsNotStartedException.java | 5 +-
.../kafka/streams/kstream/CogroupedKStream.java | 9 +
.../kafka/streams/kstream/KGroupedStream.java | 7 +
.../kafka/streams/kstream/SlidingWindows.java | 140 ++++
.../kstream/internals/CogroupedKStreamImpl.java | 14 +
.../internals/CogroupedStreamAggregateBuilder.java | 75 +-
.../kstream/internals/KGroupedStreamImpl.java | 16 +
.../internals/KStreamSlidingWindowAggregate.java | 307 ++++++++
.../SessionWindowedCogroupedKStreamImpl.java | 42 +-
...va => SlidingWindowedCogroupedKStreamImpl.java} | 118 ++--
...amImpl.java => SlidingWindowedKStreamImpl.java} | 144 ++--
.../TimeWindowedCogroupedKStreamImpl.java | 52 +-
.../kstream/internals/TimeWindowedKStreamImpl.java | 45 +-
.../internals/graph/GraphGraceSearchUtil.java | 6 +
.../streams/processor/api/ProcessorContext.java | 3 +-
.../internals/AbstractReadOnlyDecorator.java | 36 +-
.../internals/AbstractReadWriteDecorator.java | 32 +-
.../streams/processor/internals/StreamThread.java | 2 +-
.../internals/StreamsRebalanceListener.java | 8 +-
.../internals/metrics/StreamsMetricsImpl.java | 89 ++-
.../processor/internals/metrics/TaskMetrics.java | 2 +-
.../kafka/streams/state/ReadOnlyWindowStore.java | 173 +++--
.../apache/kafka/streams/state/WindowStore.java | 115 ++-
.../AbstractMergedSortedCacheStoreIterator.java | 1 -
.../AbstractRocksDBSegmentedBytesStore.java | 82 ++-
.../streams/state/internals/AbstractSegments.java | 29 +-
.../state/internals/CachingWindowStore.java | 241 +++++--
.../internals/ChangeLoggingWindowBytesStore.java | 31 +-
.../internals/CompositeReadOnlyWindowStore.java | 112 ++-
.../InMemoryTimeOrderedKeyValueBuffer.java | 2 +-
.../state/internals/InMemoryWindowStore.java | 177 ++++-
.../streams/state/internals/KeyValueSegments.java | 2 +-
.../MergedSortedCacheWindowStoreIterator.java | 5 +-
...rgedSortedCacheWindowStoreKeyValueIterator.java | 5 +-
.../state/internals/MeteredKeyValueStore.java | 4 +-
.../state/internals/MeteredSessionStore.java | 4 +-
.../state/internals/MeteredWindowStore.java | 52 +-
.../state/internals/ReadOnlyWindowStoreFacade.java | 58 +-
.../streams/state/internals/RocksDBStore.java | 2 +-
.../state/internals/RocksDBTimestampedStore.java | 1 -
.../state/internals/RocksDBWindowStore.java | 36 +-
.../kafka/streams/state/internals/Segment.java | 4 -
.../streams/state/internals/SegmentIterator.java | 19 +-
.../state/internals/SegmentedBytesStore.java | 37 +-
.../kafka/streams/state/internals/Segments.java | 4 +-
.../streams/state/internals/SessionKeySchema.java | 5 +-
.../state/internals/TimestampedSegments.java | 2 +-
.../internals/TimestampedWindowStoreBuilder.java | 42 +-
.../streams/state/internals/WindowKeySchema.java | 9 +-
.../WindowToTimestampedWindowByteStoreAdapter.java | 74 +-
.../state/internals/metrics/RocksDBMetrics.java | 374 +++++++++-
.../internals/metrics/RocksDBMetricsRecorder.java | 258 ++++++-
.../state/internals/metrics/StateStoreMetrics.java | 25 +-
.../KStreamAggregationIntegrationTest.java | 204 ++++++
.../integration/RocksDBMetricsIntegrationTest.java | 47 +-
.../kafka/streams/kstream/SessionWindowsTest.java | 9 +-
.../kafka/streams/kstream/SlidingWindowsTest.java | 82 +++
.../internals/CogroupedKStreamImplTest.java | 55 +-
.../kstream/internals/KGroupedStreamImplTest.java | 317 ++++++---
.../KStreamSlidingWindowAggregateTest.java | 692 ++++++++++++++++++
...> SlidingWindowedCogroupedKStreamImplTest.java} | 258 +++----
.../internals/SlidingWindowedKStreamImplTest.java | 439 ++++++++++++
.../kstream/internals/SuppressScenarioTest.java | 84 +++
...HandlingSourceTopicDeletionIntegrationTest.java | 114 +++
.../internals/StreamsRebalanceListenerTest.java | 74 ++
.../internals/metrics/StreamsMetricsImplTest.java | 470 ++++++++-----
.../internals/metrics/TaskMetricsTest.java | 1 -
.../kafka/streams/state/NoOpWindowStore.java | 45 +-
.../state/internals/AbstractKeyValueStoreTest.java | 29 +-
.../internals/AbstractWindowBytesStoreTest.java | 188 ++++-
.../state/internals/CachingWindowStoreTest.java | 234 ++++++-
.../ChangeLoggingWindowBytesStoreTest.java | 25 +-
.../CompositeReadOnlyWindowStoreTest.java | 264 +++++--
.../DelegatingPeekingKeyValueIteratorTest.java | 19 +-
.../state/internals/InMemoryWindowStoreTest.java | 25 +-
.../state/internals/KeyValueSegmentTest.java | 2 +-
.../state/internals/KeyValueSegmentsTest.java | 41 +-
...dSortedCacheWrappedWindowStoreIteratorTest.java | 106 ++-
...acheWrappedWindowStoreKeyValueIteratorTest.java | 76 +-
.../state/internals/MeteredWindowStoreTest.java | 11 +-
.../state/internals/ReadOnlyWindowStoreStub.java | 207 +++++-
.../streams/state/internals/RocksDBStoreTest.java | 92 ++-
.../state/internals/SegmentIteratorTest.java | 90 ++-
.../streams/state/internals/ThreadCacheTest.java | 10 +-
.../TimestampedKeyValueStoreBuilderTest.java | 21 +-
.../state/internals/TimestampedSegmentTest.java | 2 +-
.../state/internals/TimestampedSegmentsTest.java | 41 +-
.../TimestampedWindowStoreBuilderTest.java | 17 +-
.../state/internals/WindowKeySchemaTest.java | 64 +-
.../metrics/RocksDBMetricsRecorderGaugesTest.java | 267 +++++++
.../metrics/RocksDBMetricsRecorderTest.java | 169 ++++-
.../internals/metrics/RocksDBMetricsTest.java | 275 +++++++-
.../internals/metrics/StateStoreMetricsTest.java | 22 +-
.../apache/kafka/test/SegmentedBytesStoreStub.java | 112 ---
.../org/apache/kafka/test/StreamsTestUtils.java | 7 +-
.../org/apache/kafka/streams/scala/Serdes.scala | 4 +
.../streams/scala/kstream/KGroupedStream.scala | 3 +-
.../scala/kstream/SessionWindowedKStream.scala | 3 +-
.../scala/kstream/TimeWindowedKStream.scala | 3 +-
.../streams/scala/{ => serialization}/Serdes.scala | 50 +-
...bleJoinScalaIntegrationTestImplicitSerdes.scala | 19 +-
.../apache/kafka/streams/scala/TopologyTest.scala | 43 +-
.../apache/kafka/streams/scala/WordCountTest.scala | 11 +-
.../kafka/streams/scala/kstream/ConsumedTest.scala | 20 +-
.../kafka/streams/scala/kstream/GroupedTest.scala | 12 +-
.../kafka/streams/scala/kstream/JoinedTest.scala | 16 +-
.../kafka/streams/scala/kstream/KStreamTest.scala | 2 +-
.../kafka/streams/scala/kstream/KTableTest.scala | 2 +-
.../streams/scala/kstream/MaterializedTest.scala | 23 +-
.../kafka/streams/scala/kstream/ProducedTest.scala | 12 +-
.../streams/scala/kstream/RepartitionedTest.scala | 24 +-
.../streams/scala/kstream/StreamJoinedTest.scala | 22 +-
.../kafka/streams/internals/WindowStoreFacade.java | 25 +
tests/kafkatest/services/kafka/kafka.py | 21 +-
.../kafkatest/services/security/security_config.py | 44 +-
.../templates/admin_client_as_broker_jaas.conf | 16 +-
tests/kafkatest/tests/client/quota_test.py | 4 +-
tests/kafkatest/tests/core/upgrade_test.py | 2 +-
tests/kafkatest/version.py | 11 +-
206 files changed, 11736 insertions(+), 2793 deletions(-)
create mode 100644 Jenkinsfile
copy clients/src/main/java/org/apache/kafka/clients/admin/{ListPartitionReassignmentsOptions.java => AlterUserScramCredentialsOptions.java} (81%)
copy clients/src/main/java/org/apache/kafka/clients/admin/{DeleteConsumerGroupsResult.java => AlterUserScramCredentialsResult.java} (60%)
copy clients/src/main/java/org/apache/kafka/clients/admin/{AlterPartitionReassignmentsOptions.java => DescribeUserScramCredentialsOptions.java} (80%)
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/DescribeUserScramCredentialsResult.java
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ScramCredentialInfo.java
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ScramMechanism.java
copy clients/src/main/java/org/apache/kafka/clients/admin/{RenewDelegationTokenOptions.java => UserScramCredentialAlteration.java} (58%)
copy clients/src/main/java/org/apache/kafka/clients/admin/{DeletedRecords.java => UserScramCredentialDeletion.java} (55%)
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/UserScramCredentialUpsertion.java
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/UserScramCredentialsDescription.java
create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/DuplicateResourceException.java
create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/ResourceNotFoundException.java
copy clients/src/main/java/org/apache/kafka/common/errors/{DelegationTokenExpiredException.java => UnacceptableCredentialException.java} (61%)
create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java
copy clients/src/main/java/org/apache/kafka/common/requests/{ListPartitionReassignmentsResponse.java => AlterUserScramCredentialsResponse.java} (60%)
create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java
copy clients/src/main/java/org/apache/kafka/common/requests/{ListPartitionReassignmentsResponse.java => DescribeUserScramCredentialsResponse.java} (59%)
create mode 100644 clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json
copy clients/src/main/resources/common/message/{DeleteGroupsResponse.json => AlterUserScramCredentialsResponse.json} (65%)
copy clients/src/main/resources/common/message/{DeleteGroupsRequest.json => DescribeUserScramCredentialsRequest.json} (71%)
copy clients/src/main/resources/common/message/{DescribeClientQuotasResponse.json => DescribeUserScramCredentialsResponse.json} (51%)
create mode 100644 clients/src/test/java/org/apache/kafka/clients/admin/DescribeUserScramCredentialsResultTest.java
create mode 100644 core/src/test/scala/unit/kafka/admin/UserScramCredentialsCommandTest.scala
create mode 100644 core/src/test/scala/unit/kafka/server/AlterUserScramCredentialsRequestTest.scala
create mode 100644 core/src/test/scala/unit/kafka/server/DescribeUserScramCredentialsRequestTest.scala
create mode 100644 generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
copy generator/src/main/java/org/apache/kafka/message/{TypeClassGenerator.java => MessageClassGenerator.java} (66%)
copy clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java => streams/src/main/java/org/apache/kafka/streams/errors/MissingSourceTopicException.java (78%)
create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/SlidingWindows.java
create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSlidingWindowAggregate.java
copy streams/src/main/java/org/apache/kafka/streams/kstream/internals/{TimeWindowedCogroupedKStreamImpl.java => SlidingWindowedCogroupedKStreamImpl.java} (50%)
copy streams/src/main/java/org/apache/kafka/streams/kstream/internals/{TimeWindowedKStreamImpl.java => SlidingWindowedKStreamImpl.java} (61%)
create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/SlidingWindowsTest.java
create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSlidingWindowAggregateTest.java
copy streams/src/test/java/org/apache/kafka/streams/kstream/internals/{TimeWindowedCogroupedKStreamImplTest.java => SlidingWindowedCogroupedKStreamImplTest.java} (53%)
create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/SlidingWindowedKStreamImplTest.java
create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/HandlingSourceTopicDeletionIntegrationTest.java
create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListenerTest.java
create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderGaugesTest.java
delete mode 100644 streams/src/test/java/org/apache/kafka/test/SegmentedBytesStoreStub.java
copy streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/{ => serialization}/Serdes.scala (63%)
copy core/src/test/scala/kafka/server/BrokerMetadataCheckpointTest.scala => tests/kafkatest/services/security/templates/admin_client_as_broker_jaas.conf (72%)