You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by vv...@apache.org on 2020/09/09 21:40:59 UTC
[kafka] branch kip-478-part-4 updated (34d0db7 -> d3fad8b)
This is an automated email from the ASF dual-hosted git repository.
vvcephei pushed a change to branch kip-478-part-4
in repository https://gitbox.apache.org/repos/asf/kafka.git.
discard 34d0db7 finalizing the proposal and renaming the adapters for clarity
discard 08a8bf0 tweak
discard 662f0ae update
discard 396e1ea propose StateStoreContext instead
discard 92ff47d javadoc fix
discard eb55d70 Convert test-utils and StateStore
omit 3b81208 drop defaulted method
omit f77030c fix
omit de3fab1 cleanup
omit f58e275 roll back statestore change
omit 8099dd0 Port addProcessor
add d2c978c MINOR: fix JavaDoc (#9217)
add b6ba674 KAFKA-10384: Separate converters from generated messages (#9194)
add 9da32b6 KAFKA-9924: Add RocksDB metric num-entries-active-mem-table (#9177)
add 85b6545 KAFKA-5636: SlidingWindows (KIP-450) (#9039)
add d0111d3 KAFKA-10020: Create a new version of a scala Serdes without name clash (KIP-616) (#8955)
add 241e144 KAFKA-10444: Configure PR builds via Jenkinsfile (#9238)
add 59206b8 https://issues.apache.org/jira/browse/KAFKA-10456 (#9240)
add 6e8a01e KAFKA-10098: Remove unnecessary escaping in regular expression. (#8798)
add 55c1ab6 KAFKA-10272: Add IBM i support to "stop" scripts (#9023)
add fd02c8f MINOR: Include call name in TimeoutException (#8050)
add 4662ed4 MINOR: Fix build scala 2.12 build after KAFKA-10020 (#9245)
add 4f06d9e KAFKA-9929: Support backward iterator on WindowStore (#9138)
add 24b03a6 MINOR: Fix message count for sliding windows test (#9248)
add c04000c KAFKA-9924: Add remaining property-based RocksDB metrics as described in KIP-607 (#9232)
add 1e31354 KAFKA-10355: Throw error when source topic was deleted (#9191)
add 9c8501f MINOR: Record all poll invocations (#9234)
add 23cade8 KAFKA-10314: KafkaStorageException on reassignment when offline log directories exist (#9122)
add c6d135e KAFKA-10133: MM2 readme update on config (#9215)
add e8524cc KAFKA-10259: KIP-554 Broker-side SCRAM Config API (#9032)
add d7f3505 MINOR: Update scala default version in readme (#9260)
add d252185 KAFKA-10432; LeaderEpochCache is incorrectly recovered for leader epoch 0 (#9219)
add 4ea709d KAFKA-10403: Replace Scala collection by Java collection in Log4jController (#9182)
add 8e9cf0e MINOR: mirror integration tests should not call System.exit (#9200)
add 09d1498 KAFKA-10436: Implement KIP-478 Topology changes (#9221)
add dd2b9ec KAFKA-5636: Improve handling of "early" records in sliding windows (#9157)
add d3fad8b Port addProcessor
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (34d0db7)
\
N -- N -- N refs/heads/kip-478-part-4 (d3fad8b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
Jenkinsfile | 169 ++++
README.md | 3 +-
bin/kafka-server-stop.sh | 5 +-
bin/zookeeper-server-stop.sh | 5 +-
build.gradle | 33 +-
checkstyle/import-control.xml | 2 +
checkstyle/suppressions.xml | 9 +-
.../java/org/apache/kafka/clients/admin/Admin.java | 92 +++
....java => AlterUserScramCredentialsOptions.java} | 6 +-
...t.java => AlterUserScramCredentialsResult.java} | 28 +-
...va => DescribeUserScramCredentialsOptions.java} | 6 +-
.../admin/DescribeUserScramCredentialsResult.java | 150 ++++
.../kafka/clients/admin/KafkaAdminClient.java | 190 ++++-
.../kafka/clients/admin/ScramCredentialInfo.java | 78 ++
.../apache/kafka/clients/admin/ScramMechanism.java | 81 ++
...ons.java => UserScramCredentialAlteration.java} | 28 +-
...cords.java => UserScramCredentialDeletion.java} | 29 +-
.../admin/UserScramCredentialUpsertion.java | 100 +++
.../admin/UserScramCredentialsDescription.java | 82 ++
.../common/errors/DuplicateResourceException.java | 77 ++
.../common/errors/ResourceNotFoundException.java | 76 ++
...n.java => UnacceptableCredentialException.java} | 23 +-
.../org/apache/kafka/common/protocol/ApiKeys.java | 10 +-
.../org/apache/kafka/common/protocol/Errors.java | 8 +-
.../org/apache/kafka/common/protocol/Message.java | 42 -
.../kafka/common/requests/AbstractRequest.java | 4 +
.../kafka/common/requests/AbstractResponse.java | 4 +
.../requests/AlterUserScramCredentialsRequest.java | 102 +++
...java => AlterUserScramCredentialsResponse.java} | 24 +-
.../DescribeUserScramCredentialsRequest.java | 87 ++
...a => DescribeUserScramCredentialsResponse.java} | 24 +-
.../security/scram/internals/ScramFormatter.java | 30 +-
.../security/scram/internals/ScramSaslClient.java | 4 +-
.../security/scram/internals/ScramSaslServer.java | 2 +-
.../message/AlterUserScramCredentialsRequest.json | 44 ++
...json => AlterUserScramCredentialsResponse.json} | 23 +-
...on => DescribeUserScramCredentialsRequest.json} | 18 +-
...n => DescribeUserScramCredentialsResponse.json} | 40 +-
.../DescribeUserScramCredentialsResultTest.java | 118 +++
.../kafka/clients/admin/KafkaAdminClientTest.java | 154 ++++
.../kafka/clients/admin/MockAdminClient.java | 10 +
.../apache/kafka/common/message/MessageTest.java | 21 +-
.../kafka/common/protocol/ProtoUtilsTest.java | 18 +-
.../authenticator/SaslAuthenticatorTest.java | 2 +-
.../scram/internals/ScramFormatterTest.java | 12 +-
.../scram/internals/ScramMessagesTest.java | 4 +-
connect/mirror/README.md | 35 +-
.../mirror/MirrorConnectorsIntegrationTest.java | 14 +-
.../src/main/scala/kafka/admin/ConfigCommand.scala | 163 ++--
.../scala/kafka/controller/KafkaController.scala | 9 +-
core/src/main/scala/kafka/log/LogSegment.scala | 2 +-
.../src/main/scala/kafka/server/AdminManager.scala | 253 +++++-
core/src/main/scala/kafka/server/KafkaApis.scala | 35 +-
.../main/scala/kafka/tools/ConsoleProducer.scala | 2 +-
.../main/scala/kafka/utils/Log4jController.scala | 5 +-
.../kafka/api/CustomQuotaCallbackTest.scala | 20 +-
.../DelegationTokenEndToEndAuthorizationTest.scala | 73 +-
.../kafka/api/EndToEndAuthorizationTest.scala | 20 +-
.../SaslClientsWithInvalidCredentialsTest.scala | 7 +-
.../SaslScramSslEndToEndAuthorizationTest.scala | 6 +-
.../scala/integration/kafka/api/SaslSetup.scala | 43 +
.../server/DynamicBrokerReconfigurationTest.scala | 14 +-
.../kafka/tools/MirrorMakerIntegrationTest.scala | 23 +-
core/src/test/scala/kafka/utils/LoggingTest.scala | 8 +
.../scala/unit/kafka/admin/ConfigCommandTest.scala | 125 ++-
.../admin/UserScramCredentialsCommandTest.scala | 137 ++++
.../controller/ControllerIntegrationTest.scala | 43 +-
.../kafka/integration/KafkaServerTestHarness.scala | 9 +
.../test/scala/unit/kafka/log/LogSegmentTest.scala | 56 +-
.../AlterUserScramCredentialsRequestTest.scala | 433 ++++++++++
.../kafka/server/ClientQuotasRequestTest.scala | 13 +-
.../DescribeUserScramCredentialsRequestTest.scala | 149 ++++
.../unit/kafka/server/LogDirFailureTest.scala | 38 +-
.../scala/unit/kafka/server/RequestQuotaTest.scala | 6 +
.../scala/unit/kafka/utils/JaasTestUtils.scala | 47 +-
.../test/scala/unit/kafka/utils/TestUtils.scala | 33 +-
docs/streams/developer-guide/dsl-api.html | 12 +-
.../java/org/apache/kafka/message/FieldSpec.java | 340 ++++++++
.../java/org/apache/kafka/message/FieldType.java | 65 ++
.../kafka/message/JsonConverterGenerator.java | 412 ++++++++++
...ssGenerator.java => MessageClassGenerator.java} | 21 +-
.../apache/kafka/message/MessageDataGenerator.java | 770 ++----------------
.../org/apache/kafka/message/MessageGenerator.java | 91 ++-
.../java/org/apache/kafka/message/MessageSpec.java | 2 +-
.../org/apache/kafka/message/SchemaGenerator.java | 2 +-
.../org/apache/kafka/message/StructRegistry.java | 74 +-
.../errors/MissingSourceTopicException.java | 9 +-
.../streams/errors/StreamsNotStartedException.java | 5 +-
.../kafka/streams/kstream/CogroupedKStream.java | 9 +
.../kafka/streams/kstream/KGroupedStream.java | 7 +
.../kafka/streams/kstream/SlidingWindows.java | 140 ++++
.../kstream/internals/CogroupedKStreamImpl.java | 14 +
.../internals/CogroupedStreamAggregateBuilder.java | 75 +-
.../kstream/internals/KGroupedStreamImpl.java | 16 +
.../internals/KStreamSlidingWindowAggregate.java | 444 +++++++++++
.../SessionWindowedCogroupedKStreamImpl.java | 42 +-
...va => SlidingWindowedCogroupedKStreamImpl.java} | 118 +--
...amImpl.java => SlidingWindowedKStreamImpl.java} | 144 ++--
.../TimeWindowedCogroupedKStreamImpl.java | 52 +-
.../kstream/internals/TimeWindowedKStreamImpl.java | 45 +-
.../internals/graph/GraphGraceSearchUtil.java | 6 +
.../streams/processor/api/ProcessorContext.java | 3 +-
.../internals/AbstractReadOnlyDecorator.java | 36 +-
.../internals/AbstractReadWriteDecorator.java | 32 +-
.../streams/processor/internals/StreamThread.java | 2 +-
.../internals/StreamsRebalanceListener.java | 8 +-
.../internals/metrics/StreamsMetricsImpl.java | 89 ++-
.../processor/internals/metrics/TaskMetrics.java | 2 +-
.../kafka/streams/state/ReadOnlyWindowStore.java | 173 ++--
.../apache/kafka/streams/state/WindowStore.java | 115 ++-
.../AbstractMergedSortedCacheStoreIterator.java | 1 -
.../AbstractRocksDBSegmentedBytesStore.java | 82 +-
.../streams/state/internals/AbstractSegments.java | 29 +-
.../state/internals/CachingWindowStore.java | 241 ++++--
.../internals/ChangeLoggingWindowBytesStore.java | 31 +-
.../internals/CompositeReadOnlyWindowStore.java | 112 ++-
.../InMemoryTimeOrderedKeyValueBuffer.java | 2 +-
.../state/internals/InMemoryWindowStore.java | 177 ++++-
.../streams/state/internals/KeyValueSegments.java | 2 +-
.../MergedSortedCacheWindowStoreIterator.java | 5 +-
...rgedSortedCacheWindowStoreKeyValueIterator.java | 5 +-
.../state/internals/MeteredKeyValueStore.java | 4 +-
.../state/internals/MeteredSessionStore.java | 4 +-
.../state/internals/MeteredWindowStore.java | 52 +-
.../state/internals/ReadOnlyWindowStoreFacade.java | 58 +-
.../streams/state/internals/RocksDBStore.java | 2 +-
.../state/internals/RocksDBTimestampedStore.java | 1 -
.../state/internals/RocksDBWindowStore.java | 36 +-
.../kafka/streams/state/internals/Segment.java | 4 -
.../streams/state/internals/SegmentIterator.java | 19 +-
.../state/internals/SegmentedBytesStore.java | 37 +-
.../kafka/streams/state/internals/Segments.java | 4 +-
.../streams/state/internals/SessionKeySchema.java | 5 +-
.../state/internals/TimestampedSegments.java | 2 +-
.../internals/TimestampedWindowStoreBuilder.java | 42 +-
.../streams/state/internals/WindowKeySchema.java | 9 +-
.../WindowToTimestampedWindowByteStoreAdapter.java | 74 +-
.../state/internals/metrics/RocksDBMetrics.java | 374 ++++++++-
.../internals/metrics/RocksDBMetricsRecorder.java | 258 +++++-
.../state/internals/metrics/StateStoreMetrics.java | 25 +-
.../KStreamAggregationIntegrationTest.java | 204 +++++
.../integration/RocksDBMetricsIntegrationTest.java | 47 +-
.../kafka/streams/kstream/SessionWindowsTest.java | 9 +-
.../kafka/streams/kstream/SlidingWindowsTest.java | 82 ++
.../internals/CogroupedKStreamImplTest.java | 55 +-
.../kstream/internals/KGroupedStreamImplTest.java | 317 +++++---
.../KStreamSlidingWindowAggregateTest.java | 880 +++++++++++++++++++++
...> SlidingWindowedCogroupedKStreamImplTest.java} | 258 +++---
.../internals/SlidingWindowedKStreamImplTest.java | 439 ++++++++++
.../kstream/internals/SuppressScenarioTest.java | 84 ++
...HandlingSourceTopicDeletionIntegrationTest.java | 114 +++
.../internals/StreamsRebalanceListenerTest.java | 74 ++
.../internals/metrics/StreamsMetricsImplTest.java | 470 +++++++----
.../internals/metrics/TaskMetricsTest.java | 1 -
.../kafka/streams/state/NoOpWindowStore.java | 45 +-
.../state/internals/AbstractKeyValueStoreTest.java | 29 +-
.../internals/AbstractWindowBytesStoreTest.java | 188 ++++-
.../state/internals/CachingWindowStoreTest.java | 234 +++++-
.../ChangeLoggingWindowBytesStoreTest.java | 25 +-
.../CompositeReadOnlyWindowStoreTest.java | 264 ++++++-
.../DelegatingPeekingKeyValueIteratorTest.java | 19 +-
.../state/internals/InMemoryWindowStoreTest.java | 25 +-
.../state/internals/KeyValueSegmentTest.java | 2 +-
.../state/internals/KeyValueSegmentsTest.java | 41 +-
...dSortedCacheWrappedWindowStoreIteratorTest.java | 106 ++-
...acheWrappedWindowStoreKeyValueIteratorTest.java | 76 +-
.../state/internals/MeteredWindowStoreTest.java | 11 +-
.../state/internals/ReadOnlyWindowStoreStub.java | 207 ++++-
.../streams/state/internals/RocksDBStoreTest.java | 92 ++-
.../state/internals/SegmentIteratorTest.java | 90 ++-
.../streams/state/internals/ThreadCacheTest.java | 10 +-
.../TimestampedKeyValueStoreBuilderTest.java | 21 +-
.../state/internals/TimestampedSegmentTest.java | 2 +-
.../state/internals/TimestampedSegmentsTest.java | 41 +-
.../TimestampedWindowStoreBuilderTest.java | 17 +-
.../state/internals/WindowKeySchemaTest.java | 64 +-
.../metrics/RocksDBMetricsRecorderGaugesTest.java | 267 +++++++
.../metrics/RocksDBMetricsRecorderTest.java | 169 +++-
.../internals/metrics/RocksDBMetricsTest.java | 275 ++++++-
.../internals/metrics/StateStoreMetricsTest.java | 22 +-
.../apache/kafka/test/SegmentedBytesStoreStub.java | 112 ---
.../org/apache/kafka/test/StreamsTestUtils.java | 7 +-
.../org/apache/kafka/streams/scala/Serdes.scala | 4 +
.../streams/scala/kstream/KGroupedStream.scala | 3 +-
.../scala/kstream/SessionWindowedKStream.scala | 3 +-
.../scala/kstream/TimeWindowedKStream.scala | 3 +-
.../streams/scala/{ => serialization}/Serdes.scala | 50 +-
...bleJoinScalaIntegrationTestImplicitSerdes.scala | 19 +-
.../apache/kafka/streams/scala/TopologyTest.scala | 43 +-
.../apache/kafka/streams/scala/WordCountTest.scala | 11 +-
.../kafka/streams/scala/kstream/ConsumedTest.scala | 20 +-
.../kafka/streams/scala/kstream/GroupedTest.scala | 12 +-
.../kafka/streams/scala/kstream/JoinedTest.scala | 16 +-
.../kafka/streams/scala/kstream/KStreamTest.scala | 2 +-
.../kafka/streams/scala/kstream/KTableTest.scala | 2 +-
.../streams/scala/kstream/MaterializedTest.scala | 23 +-
.../kafka/streams/scala/kstream/ProducedTest.scala | 12 +-
.../streams/scala/kstream/RepartitionedTest.scala | 24 +-
.../streams/scala/kstream/StreamJoinedTest.scala | 22 +-
.../kafka/streams/internals/WindowStoreFacade.java | 25 +
tests/kafkatest/services/kafka/kafka.py | 21 +-
.../kafkatest/services/security/security_config.py | 44 +-
.../templates/admin_client_as_broker_jaas.conf | 16 +-
tests/kafkatest/tests/client/quota_test.py | 4 +-
tests/kafkatest/tests/core/upgrade_test.py | 2 +-
tests/kafkatest/version.py | 11 +-
206 files changed, 12064 insertions(+), 2793 deletions(-)
create mode 100644 Jenkinsfile
copy clients/src/main/java/org/apache/kafka/clients/admin/{ListPartitionReassignmentsOptions.java => AlterUserScramCredentialsOptions.java} (81%)
copy clients/src/main/java/org/apache/kafka/clients/admin/{DeleteConsumerGroupsResult.java => AlterUserScramCredentialsResult.java} (60%)
copy clients/src/main/java/org/apache/kafka/clients/admin/{AlterPartitionReassignmentsOptions.java => DescribeUserScramCredentialsOptions.java} (80%)
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/DescribeUserScramCredentialsResult.java
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ScramCredentialInfo.java
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ScramMechanism.java
copy clients/src/main/java/org/apache/kafka/clients/admin/{RenewDelegationTokenOptions.java => UserScramCredentialAlteration.java} (58%)
copy clients/src/main/java/org/apache/kafka/clients/admin/{DeletedRecords.java => UserScramCredentialDeletion.java} (55%)
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/UserScramCredentialUpsertion.java
create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/UserScramCredentialsDescription.java
create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/DuplicateResourceException.java
create mode 100644 clients/src/main/java/org/apache/kafka/common/errors/ResourceNotFoundException.java
copy clients/src/main/java/org/apache/kafka/common/errors/{DelegationTokenExpiredException.java => UnacceptableCredentialException.java} (61%)
create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java
copy clients/src/main/java/org/apache/kafka/common/requests/{ListPartitionReassignmentsResponse.java => AlterUserScramCredentialsResponse.java} (60%)
create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java
copy clients/src/main/java/org/apache/kafka/common/requests/{ListPartitionReassignmentsResponse.java => DescribeUserScramCredentialsResponse.java} (59%)
create mode 100644 clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json
copy clients/src/main/resources/common/message/{DeleteGroupsResponse.json => AlterUserScramCredentialsResponse.json} (65%)
copy clients/src/main/resources/common/message/{DeleteGroupsRequest.json => DescribeUserScramCredentialsRequest.json} (71%)
copy clients/src/main/resources/common/message/{DescribeClientQuotasResponse.json => DescribeUserScramCredentialsResponse.json} (51%)
create mode 100644 clients/src/test/java/org/apache/kafka/clients/admin/DescribeUserScramCredentialsResultTest.java
create mode 100644 core/src/test/scala/unit/kafka/admin/UserScramCredentialsCommandTest.scala
create mode 100644 core/src/test/scala/unit/kafka/server/AlterUserScramCredentialsRequestTest.scala
create mode 100644 core/src/test/scala/unit/kafka/server/DescribeUserScramCredentialsRequestTest.scala
create mode 100644 generator/src/main/java/org/apache/kafka/message/JsonConverterGenerator.java
copy generator/src/main/java/org/apache/kafka/message/{TypeClassGenerator.java => MessageClassGenerator.java} (66%)
copy clients/src/main/java/org/apache/kafka/common/errors/GroupSubscribedToTopicException.java => streams/src/main/java/org/apache/kafka/streams/errors/MissingSourceTopicException.java (78%)
create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/SlidingWindows.java
create mode 100644 streams/src/main/java/org/apache/kafka/streams/kstream/internals/KStreamSlidingWindowAggregate.java
copy streams/src/main/java/org/apache/kafka/streams/kstream/internals/{TimeWindowedCogroupedKStreamImpl.java => SlidingWindowedCogroupedKStreamImpl.java} (50%)
copy streams/src/main/java/org/apache/kafka/streams/kstream/internals/{TimeWindowedKStreamImpl.java => SlidingWindowedKStreamImpl.java} (61%)
create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/SlidingWindowsTest.java
create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamSlidingWindowAggregateTest.java
copy streams/src/test/java/org/apache/kafka/streams/kstream/internals/{TimeWindowedCogroupedKStreamImplTest.java => SlidingWindowedCogroupedKStreamImplTest.java} (53%)
create mode 100644 streams/src/test/java/org/apache/kafka/streams/kstream/internals/SlidingWindowedKStreamImplTest.java
create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/HandlingSourceTopicDeletionIntegrationTest.java
create mode 100644 streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsRebalanceListenerTest.java
create mode 100644 streams/src/test/java/org/apache/kafka/streams/state/internals/metrics/RocksDBMetricsRecorderGaugesTest.java
delete mode 100644 streams/src/test/java/org/apache/kafka/test/SegmentedBytesStoreStub.java
copy streams/streams-scala/src/main/scala/org/apache/kafka/streams/scala/{ => serialization}/Serdes.scala (63%)
copy core/src/test/scala/kafka/server/BrokerMetadataCheckpointTest.scala => tests/kafkatest/services/security/templates/admin_client_as_broker_jaas.conf (72%)