You are viewing a plain text version of this content. The canonical link for it is here.
Posted to jira@kafka.apache.org by GitBox <gi...@apache.org> on 2021/04/21 12:42:13 UTC
[GitHub] [kafka] satishd opened a new pull request #10579: KAFKA-9555 Initial version of default RLMM implementation.
satishd opened a new pull request #10579:
URL: https://github.com/apache/kafka/pull/10579
KAFKA-9555 Initial version of default RLMM implementation.
This is still in draft mode.
This includes changes containing default RLMM configs, RLMM implementation, producer/consumer managers.
### Committer Checklist (excluded from commit message)
- [ ] Verify design and implementation
- [ ] Verify test coverage and CI build status
- [ ] Verify documentation (including upgrade notes)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r671516460
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
What about client security related properties? It's weird that we pick up "bootstrap.servers" from one prefix, but the corresponding security properties under a different prefix. If we do provide the security related properties in the same prefix, they seem to be duplicated from those under prefix REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX, REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX or REMOTE_LOG_METADATA_CONSUMER_PREFIX.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] ccding commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
ccding commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664632598
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -51,6 +58,7 @@
*/
public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
Review comment:
our of curiosity: why we have `L` here but not https://github.com/apache/kafka/pull/10579/commits/77da4ebff5228d1c8938105d50b6b15a5223b294#diff-9f35ec72323f704125b5ee6d35a230de7f28f8c448a706eec2946fa46a12cfedR63?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r670820515
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
Since bootstrap_servers is used for the internal producer/consumer, should bootstrap_servers be defined with a prefix of REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX, REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX or REMOTE_LOG_METADATA_CONSUMER_PREFIX?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.FatalExitError;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private final Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+ private volatile boolean initializationFailed;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ final NewTopic remoteLogMetadataTopicRequest = createRemoteLogMetadataTopicRequest();
+ boolean topicCreated = false;
+ long startTimeMs = time.milliseconds();
+ AdminClient adminClient = null;
+ try {
+ adminClient = AdminClient.create(rlmmConfig.producerProperties());
+
+ // Stop if it is already initialized or closing.
+ while (!(initialized.get() || closing.get())) {
+
+ // If it is timed out then raise an error to exit.
+ if (time.milliseconds() - startTimeMs > rlmmConfig.initializationRetryMaxTimeoutMs()) {
+ log.error("Timed out in initializing the resources, retried to initialize the resource for [{}] ms.",
+ rlmmConfig.initializationRetryMaxTimeoutMs());
+ initializationFailed = true;
Review comment:
Should halt the jvm in this case? Ditto below when the partition count doesn't match.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.FatalExitError;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private final Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
Review comment:
Does pendingAssignPartitions need to be synchronizedSet since it's accessed under a lock?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao merged pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao merged pull request #10579:
URL: https://github.com/apache/kafka/pull/10579
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] kowshik commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
kowshik commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r643698596
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
Review comment:
Could you pls add a comment for this class?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
+ commonClientConfigs.put(key.substring(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_PRODUCER_PREFIX)) {
+ producerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_PRODUCER_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_CONSUMER_PREFIX)) {
+ consumerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_CONSUMER_PREFIX.length()), entry.getValue());
+ }
+ }
+
+ HashMap<String, Object> allProducerConfigs = new HashMap<>(commonClientConfigs);
+ allProducerConfigs.putAll(producerOnlyConfigs);
+ producerProps = createProducerProps(allProducerConfigs);
+
+ HashMap<String, Object> allConsumerConfigs = new HashMap<>(commonClientConfigs);
+ allConsumerConfigs.putAll(consumerOnlyConfigs);
+ consumerProps = createConsumerProps(allConsumerConfigs);
+ }
+
+ public String remoteLogMetadataTopicName() {
+ return REMOTE_LOG_METADATA_TOPIC_NAME;
+ }
+
+ public int metadataTopicPartitionsCount() {
+ return metadataTopicPartitionsCount;
+ }
+
+ public long consumeWaitMs() {
+ return consumeWaitMs;
+ }
+
+ public Map<String, Object> consumerProperties() {
+ return consumerProps;
+ }
+
+ public Map<String, Object> producerProperties() {
+ return producerProps;
+ }
+
+ private Map<String, Object> createConsumerProps(HashMap<String, Object> allConsumerConfigs) {
+ Map<String, Object> props = new HashMap<>(allConsumerConfigs);
+
+ props.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientIdPrefix + "_consumer");
+ props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
+ props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG, false);
Review comment:
Hmm, why do you need this exclusion to be true?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
Review comment:
Is this the timeout for how long you'd want the client to wait to consume the message that it produces to `__remote_log_metadata` topic? If yes, then don't we want this timeout to be unlimited i.e. we wait as long as it takes to consume the published event?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
+ commonClientConfigs.put(key.substring(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_PRODUCER_PREFIX)) {
+ producerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_PRODUCER_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_CONSUMER_PREFIX)) {
+ consumerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_CONSUMER_PREFIX.length()), entry.getValue());
+ }
+ }
+
+ HashMap<String, Object> allProducerConfigs = new HashMap<>(commonClientConfigs);
+ allProducerConfigs.putAll(producerOnlyConfigs);
+ producerProps = createProducerProps(allProducerConfigs);
+
+ HashMap<String, Object> allConsumerConfigs = new HashMap<>(commonClientConfigs);
+ allConsumerConfigs.putAll(consumerOnlyConfigs);
+ consumerProps = createConsumerProps(allConsumerConfigs);
+ }
+
+ public String remoteLogMetadataTopicName() {
+ return REMOTE_LOG_METADATA_TOPIC_NAME;
+ }
+
+ public int metadataTopicPartitionsCount() {
+ return metadataTopicPartitionsCount;
+ }
+
+ public long consumeWaitMs() {
+ return consumeWaitMs;
+ }
+
+ public Map<String, Object> consumerProperties() {
+ return consumerProps;
+ }
+
+ public Map<String, Object> producerProperties() {
+ return producerProps;
+ }
+
+ private Map<String, Object> createConsumerProps(HashMap<String, Object> allConsumerConfigs) {
+ Map<String, Object> props = new HashMap<>(allConsumerConfigs);
+
+ props.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientIdPrefix + "_consumer");
+ props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
+ props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG, false);
+ props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
+ props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
+ return props;
+ }
+
+ private Map<String, Object> createProducerProps(HashMap<String, Object> allProducerConfigs) {
+ Map<String, Object> props = new HashMap<>(allProducerConfigs);
+
+ props.put(ProducerConfig.CLIENT_ID_CONFIG, clientIdPrefix + "_producer");
+ props.put(ProducerConfig.ACKS_CONFIG, "all");
+ props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 1);
+ props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
+ props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
+
+ return Collections.unmodifiableMap(props);
+ }
+
+ @Override
+ public String toString() {
+ return "TopicBasedRemoteLogMetadataManagerConfig{" +
+ "clientIdPrefix='" + clientIdPrefix + '\'' +
+ ", metadataTopicPartitionsCount=" + metadataTopicPartitionsCount +
+ ", bootstrapServers='" + bootstrapServers + '\'' +
+ ", consumeWaitMs=" + consumeWaitMs +
+ ", metadataTopicRetentionMillis=" + metadataTopicRetentionMillis +
+ ", consumerProps=" + consumerProps +
Review comment:
`consumerProps` and `producerProps` are of type `Map`, therefore the `.toString()` is probably not readable. So you'd need to convert these into a comma-separated list sth like `K1=V1,K2=V2,...Kn=Vn`.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
Review comment:
Could you pls document the state does this boolean represents?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] kowshik commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
kowshik commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r643693257
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNo, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNo >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNo +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNo, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() != null) {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
+ } else {
+ throw new KafkaException(ex);
+ }
+ } else {
+ return callback.recordMetadata();
+ }
+ }
+
+ private void ensureNotClosed() {
+ if (close) {
+ throw new IllegalStateException("This instance is already set in close state.");
Review comment:
s/set in close state/closed
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadata) throws KafkaException {
Review comment:
It appears you could eliminate the additional `topicIdPartition` parameter and instead use the value returned by `remoteLogMetadata.topicIdPartition()` API.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
Review comment:
It seems that we have internal topics specified in `org.apache.kafka.common.internals.Topic` class.
Don't we want this new internal topic to be defined in the `Topic` class, together with other internal topics?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
Review comment:
Is this the timeout for how long we want the client to wait to consume the message that it produces to `__remote_log_metadata` topic? If yes, then don't we want this timeout to be unlimited i.e. we wait as long as it takes to consume the published event?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
Review comment:
Can you pls document the state does this boolean represents?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
Review comment:
Is it useful to assert that `record.key()` is empty before the key is ignored below?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogMetadataTopicPartitioner.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+public class RemoteLogMetadataTopicPartitioner {
+ public static final Logger log = LoggerFactory.getLogger(RemoteLogMetadataTopicPartitioner.class);
+ private final int noOfMetadataTopicPartitions;
+
+ public RemoteLogMetadataTopicPartitioner(int noOfMetadataTopicPartitions) {
Review comment:
s/noOfMetadataTopicPartitions/numMetadataTopicPartitions
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemotePartitionMetadataStore.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteState;
+import org.apache.kafka.server.log.remote.storage.RemoteResourceNotFoundException;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RemotePartitionMetadataStore extends RemotePartitionMetadataEventHandler implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(RemotePartitionMetadataStore.class);
+
+ private Map<TopicIdPartition, RemotePartitionDeleteMetadata> idToPartitionDeleteMetadata =
Review comment:
It seems these 2 attributes can be marked final if you call `Map.clear()` in `close()` instead of replacing the reference.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
Review comment:
It appears this class does not have unit tests currently. Is there a plan to add unit tests?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
Review comment:
Can we add a comment for this class please?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
Review comment:
s/metadataPartitionNo/metadataPartition
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogMetadataTopicPartitioner.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+public class RemoteLogMetadataTopicPartitioner {
+ public static final Logger log = LoggerFactory.getLogger(RemoteLogMetadataTopicPartitioner.class);
+ private final int noOfMetadataTopicPartitions;
+
+ public RemoteLogMetadataTopicPartitioner(int noOfMetadataTopicPartitions) {
+ this.noOfMetadataTopicPartitions = noOfMetadataTopicPartitions;
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ Objects.requireNonNull(topicIdPartition, "TopicPartition can not be null");
+
+ int partitionNo = Utils.toPositive(Utils.murmur2(toBytes(topicIdPartition))) % noOfMetadataTopicPartitions;
Review comment:
s/partitionNo/partition
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
Review comment:
s/Millis/Ms
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
+ commonClientConfigs.put(key.substring(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_PRODUCER_PREFIX)) {
+ producerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_PRODUCER_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_CONSUMER_PREFIX)) {
+ consumerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_CONSUMER_PREFIX.length()), entry.getValue());
+ }
+ }
+
+ HashMap<String, Object> allProducerConfigs = new HashMap<>(commonClientConfigs);
+ allProducerConfigs.putAll(producerOnlyConfigs);
+ producerProps = createProducerProps(allProducerConfigs);
+
+ HashMap<String, Object> allConsumerConfigs = new HashMap<>(commonClientConfigs);
+ allConsumerConfigs.putAll(consumerOnlyConfigs);
+ consumerProps = createConsumerProps(allConsumerConfigs);
+ }
+
+ public String remoteLogMetadataTopicName() {
+ return REMOTE_LOG_METADATA_TOPIC_NAME;
+ }
+
+ public int metadataTopicPartitionsCount() {
+ return metadataTopicPartitionsCount;
+ }
+
+ public long consumeWaitMs() {
+ return consumeWaitMs;
+ }
+
+ public Map<String, Object> consumerProperties() {
+ return consumerProps;
+ }
+
+ public Map<String, Object> producerProperties() {
+ return producerProps;
+ }
+
+ private Map<String, Object> createConsumerProps(HashMap<String, Object> allConsumerConfigs) {
+ Map<String, Object> props = new HashMap<>(allConsumerConfigs);
+
+ props.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientIdPrefix + "_consumer");
+ props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
+ props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG, false);
Review comment:
Hmm, why do we need this exclusion to be true?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNo, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNo >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNo +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNo, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() != null) {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
+ } else {
Review comment:
For readability, it'll be useful to place the positive case under `if` and negative case under `else`, such as:
```
if (callback.exception() == null) {
...
} else {
...
}
```
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
+ commonClientConfigs.put(key.substring(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_PRODUCER_PREFIX)) {
+ producerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_PRODUCER_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_CONSUMER_PREFIX)) {
+ consumerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_CONSUMER_PREFIX.length()), entry.getValue());
+ }
+ }
+
+ HashMap<String, Object> allProducerConfigs = new HashMap<>(commonClientConfigs);
+ allProducerConfigs.putAll(producerOnlyConfigs);
+ producerProps = createProducerProps(allProducerConfigs);
+
+ HashMap<String, Object> allConsumerConfigs = new HashMap<>(commonClientConfigs);
+ allConsumerConfigs.putAll(consumerOnlyConfigs);
+ consumerProps = createConsumerProps(allConsumerConfigs);
+ }
+
+ public String remoteLogMetadataTopicName() {
+ return REMOTE_LOG_METADATA_TOPIC_NAME;
+ }
+
+ public int metadataTopicPartitionsCount() {
+ return metadataTopicPartitionsCount;
+ }
+
+ public long consumeWaitMs() {
+ return consumeWaitMs;
+ }
+
+ public Map<String, Object> consumerProperties() {
+ return consumerProps;
+ }
+
+ public Map<String, Object> producerProperties() {
+ return producerProps;
+ }
+
+ private Map<String, Object> createConsumerProps(HashMap<String, Object> allConsumerConfigs) {
+ Map<String, Object> props = new HashMap<>(allConsumerConfigs);
+
+ props.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientIdPrefix + "_consumer");
+ props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
+ props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG, false);
+ props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
+ props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
+ return props;
+ }
+
+ private Map<String, Object> createProducerProps(HashMap<String, Object> allProducerConfigs) {
+ Map<String, Object> props = new HashMap<>(allProducerConfigs);
+
+ props.put(ProducerConfig.CLIENT_ID_CONFIG, clientIdPrefix + "_producer");
+ props.put(ProducerConfig.ACKS_CONFIG, "all");
+ props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 1);
+ props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
+ props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
+
+ return Collections.unmodifiableMap(props);
+ }
+
+ @Override
+ public String toString() {
+ return "TopicBasedRemoteLogMetadataManagerConfig{" +
+ "clientIdPrefix='" + clientIdPrefix + '\'' +
+ ", metadataTopicPartitionsCount=" + metadataTopicPartitionsCount +
+ ", bootstrapServers='" + bootstrapServers + '\'' +
+ ", consumeWaitMs=" + consumeWaitMs +
+ ", metadataTopicRetentionMillis=" + metadataTopicRetentionMillis +
+ ", consumerProps=" + consumerProps +
Review comment:
`consumerProps` and `producerProps` are of type `Map`, therefore the `.toString()` is probably not readable. So we need to convert these into a comma-separated list sth like `K1=V1,K2=V2,...Kn=Vn`.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogMetadataTopicPartitioner.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+public class RemoteLogMetadataTopicPartitioner {
+ public static final Logger log = LoggerFactory.getLogger(RemoteLogMetadataTopicPartitioner.class);
+ private final int noOfMetadataTopicPartitions;
+
+ public RemoteLogMetadataTopicPartitioner(int noOfMetadataTopicPartitions) {
+ this.noOfMetadataTopicPartitions = noOfMetadataTopicPartitions;
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ Objects.requireNonNull(topicIdPartition, "TopicPartition can not be null");
+
+ int partitionNo = Utils.toPositive(Utils.murmur2(toBytes(topicIdPartition))) % noOfMetadataTopicPartitions;
+ log.debug("No of partitions [{}], partitionNo: [{}] for given topic: [{}]", noOfMetadataTopicPartitions, partitionNo, topicIdPartition);
Review comment:
s/No of.../Num of...
Also it feels overkill to me to log this message for each call.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
Review comment:
Do you really need this explicit lock? It seems you could just use `wait()` and `notify()` APIs on the `ConsumerTask` object instead, combined with `synchronized` keyword.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
Review comment:
Hmm, should you be setting `close` to true here? (it depends on the meaning of `close`, which I don't fully understand....)
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemotePartitionMetadataStore.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteState;
+import org.apache.kafka.server.log.remote.storage.RemoteResourceNotFoundException;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RemotePartitionMetadataStore extends RemotePartitionMetadataEventHandler implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(RemotePartitionMetadataStore.class);
+
+ private Map<TopicIdPartition, RemotePartitionDeleteMetadata> idToPartitionDeleteMetadata =
+ new ConcurrentHashMap<>();
+
+ private Map<TopicIdPartition, RemoteLogMetadataCache> idToRemoteLogMetadataCache =
+ new ConcurrentHashMap<>();
+
+ @Override
+ public void handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata) {
+ log.debug("Adding remote log segment : [{}]", remoteLogSegmentMetadata);
+
+ RemoteLogSegmentId remoteLogSegmentId = remoteLogSegmentMetadata.remoteLogSegmentId();
+
+ idToRemoteLogMetadataCache
+ .computeIfAbsent(remoteLogSegmentId.topicIdPartition(), id -> new RemoteLogMetadataCache())
+ .addCopyInProgressSegment(remoteLogSegmentMetadata);
+ }
+
+ @Override
+ public void handleRemoteLogSegmentMetadataUpdate(RemoteLogSegmentMetadataUpdate rlsmUpdate) {
+ log.debug("Updating remote log segment: [{}]", rlsmUpdate);
+ RemoteLogSegmentId remoteLogSegmentId = rlsmUpdate.remoteLogSegmentId();
+ TopicIdPartition topicIdPartition = remoteLogSegmentId.topicIdPartition();
+ RemoteLogMetadataCache remoteLogMetadataCache = idToRemoteLogMetadataCache.get(topicIdPartition);
+ if (remoteLogMetadataCache != null) {
+ try {
+ remoteLogMetadataCache.updateRemoteLogSegmentMetadata(rlsmUpdate);
+ } catch (RemoteResourceNotFoundException e) {
+ log.error("Error occurred while updating the remote logs segment.");
Review comment:
s/logs/log ?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
Review comment:
I had the same question. It appears better to just duplicate the properties.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogMetadataTopicPartitioner.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+public class RemoteLogMetadataTopicPartitioner {
+ public static final Logger log = LoggerFactory.getLogger(RemoteLogMetadataTopicPartitioner.class);
+ private final int noOfMetadataTopicPartitions;
+
+ public RemoteLogMetadataTopicPartitioner(int noOfMetadataTopicPartitions) {
+ this.noOfMetadataTopicPartitions = noOfMetadataTopicPartitions;
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ Objects.requireNonNull(topicIdPartition, "TopicPartition can not be null");
+
+ int partitionNo = Utils.toPositive(Utils.murmur2(toBytes(topicIdPartition))) % noOfMetadataTopicPartitions;
+ log.debug("No of partitions [{}], partitionNo: [{}] for given topic: [{}]", noOfMetadataTopicPartitions, partitionNo, topicIdPartition);
+ return partitionNo;
+ }
+
+ private byte[] toBytes(TopicIdPartition topicIdPartition) {
+ // We do not want to depend upon hash code generation of Uuid as that may change.
+ int hash = Objects.hash(topicIdPartition.topicId().getLeastSignificantBits(),
+ topicIdPartition.topicId().getMostSignificantBits(),
+ topicIdPartition.topicPartition().topic(),
Review comment:
Is topic necessary here, when UUID and partition is already sufficient input for the hash?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
Review comment:
Could you pls mention what state does this boolean represent?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
Review comment:
Could you pls add a comment on what state does `close` represent?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
+ assignedTopicPartitions = Collections.unmodifiableSet(updatedReassignedPartitions);
+ assignedMetaPartitions = Collections.unmodifiableSet(updatedAssignedMetaPartitions);
Review comment:
It appears that once this logic is implemented, there is probably no need to wait for `maybeWaitForPartitionsAssignment`. The reason is that whenever a partition is assigned, we will bootstrap the remote state by consuming from the beginning.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
Review comment:
I agree with the question here. This can become more expensive than it seems. The alternative is to pursue an asynchronous notification model to improve the throughput.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r665555878
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ *
+ * Caller of this class should take care of not sending messages once the closing of this instance is initiated.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not.
+ private final AtomicBoolean closing = new AtomicBoolean();
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
+ " must be less than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNum, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() == null) {
+ return callback.recordMetadata();
+ } else {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
+ } else {
+ throw new KafkaException(ex);
+ }
+ }
+ }
+
+ public void close() {
+ // If it is already closed, return from here.
+ if (!closing.compareAndSet(false, true)) {
Review comment:
Good catch, we do not really need this check here as it is already guarded in `TopicBasedRemoteLogMetadataManager`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r665259927
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
@junrao The reason why we need to initialize in a different thread here is that RLMM will be created and configure() will be called before the broker starts accepting the requests. So, we can not call topic creation requests in the same thread as the brokers are not yet up.
Another way to do this is to have this topic as an internal topic and it will be auto created whenever it is accessed. afaik, creating producer and consumer instances can be done without the brokers up and running and they will not trigger a request to auto creation of remote log metadata topic. Consumer assignment will send a metadata request which will trigger auto creation of topics. This assignment on consumer is done only when the broker receives leader and isr updates from the controller. In the current PR, I will make the existing producer manager and consumer manager in configure() and have the topic creation done in the tests.
I will have a quick follow-up PR with internal topic changes and remove the topic creation code from tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r634023546
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
Review comment:
`Config` definition will be moved to `KafkaConfig` later when default RLMM is integrated with the broker.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664866782
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 100;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. If it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean closing = false;
+ // It indicates whether the consumer needs to assign the partitions or not. This is set when it is
+ // determined that the consumer needs to be assigned with the updated partitions.
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ Objects.requireNonNull(consumer);
+ Objects.requireNonNull(remotePartitionMetadataEventHandler);
+ Objects.requireNonNull(topicPartitioner);
+
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!closing) {
+ maybeWaitForPartitionsAssignment();
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofMillis(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", closing, e);
+ } finally {
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+
+ private void maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ // If it is closing, return immediately. This should be inside the assignPartitionsLock as the closing is updated
+ // in close() method with in the same lock to avoid any race conditions.
+ if (closing) {
+ return;
+ }
+
+ while (assignedMetaPartitions.isEmpty() && !closing) {
Review comment:
The test for closing seem unnecessary since closing can't change while synchronized on assignPartitionsLock.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
Is this comment addressed?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
> It was required to be retried until the topic is successfully created. I added the logic to check for topic creation too.
I am still not sure why we need to initialize in a separate thread. If we can't create the metadata topic or instantiate the producer/consumer due to wrong configurations, we want to fail fast by throwing an error to shut down the broker.
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogSegmentLifecycleManager.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Optional;
+
+/**
+ * This interface defines the lifecycle methods for {@code RemoteLogSegmentMetadata}. {@link RemoteLogSegmentLifecycleTest} tests
+ * different implementations of this interface. This is responsible for managing all the segments for a given {@code topicIdPartition}
+ * registered with {@link #initialize(TopicIdPartition)}.
+ */
+public interface RemoteLogSegmentLifecycleManager extends Closeable {
Review comment:
Is RemoteLogSegmentLifecycleManager used for tests only? If so, could we move it under tests?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
Review comment:
Hmm, why is this needed since initializeResources() does this already?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ final NewTopic remoteLogMetadataTopicRequest = createRemoteLogMetadataTopicRequest();
+
+ // Stop if it is already initialized or closing.
+ while (!(initialized.get() || closing.get())) {
+ // There were dead locks observed when the remote log metadata topic created as part of on internal
+ // topic creation(with auto create enabled) when multiple brokers were getting started and RLMM creating
+ // the respective producer and consumer instances.
+ if (!createTopic(remoteLogMetadataTopicRequest)) {
+ // try to create the topic again if it could not be created.
+ log.info("Sleep for : {} ms before it is retried again.", INITIALIZATION_RETRY_INTERVAL_MS);
+ Utils.sleep(INITIALIZATION_RETRY_INTERVAL_MS);
+ } else {
+ // Create producer and consumer managers.
+ lock.writeLock().lock();
+ try {
+ producerManager = new ProducerManager(rlmmConfig, rlmmTopicPartitioner);
+ consumerManager = new ConsumerManager(rlmmConfig, remotePartitionMetadataStore, rlmmTopicPartitioner, time);
+ consumerManager.startConsumerThread();
+
+ if (!pendingAssignPartitions.isEmpty()) {
+ consumerManager.addAssignmentsForPartitions(pendingAssignPartitions);
+ pendingAssignPartitions.clear();
+ }
+
+ initialized.set(true);
+ log.info("Initialized resources successfully.");
+ } catch (Exception e) {
+ log.error("Encountered error while initializing producer/consumer", e);
+ return;
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+ }
+ }
+
+ private NewTopic createRemoteLogMetadataTopicRequest() {
+ Map<String, String> topicConfigs = new HashMap<>();
+ topicConfigs.put(TopicConfig.RETENTION_MS_CONFIG, Long.toString(rlmmConfig.metadataTopicRetentionMs()));
+ topicConfigs.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
+ return new NewTopic(rlmmConfig.remoteLogMetadataTopicName(),
+ rlmmConfig.metadataTopicPartitionsCount(),
+ rlmmConfig.metadataTopicReplicationFactor()).configs(topicConfigs);
+ }
+
+ /**
+ * @param topic topic to be created.
+ * @return Returns true if the topic already exists or it is created successfully.
+ */
+ private boolean createTopic(NewTopic topic) {
+ boolean topicCreated = false;
+ AdminClient adminClient = null;
+ try {
+ adminClient = AdminClient.create(rlmmConfig.consumerProperties());
+ adminClient.createTopics(Collections.singleton(topic)).all().get();
+ topicCreated = true;
+ } catch (Exception e) {
+ if (e.getCause() instanceof TopicExistsException) {
+ log.info("Topic [{}] already exists", topic.name());
+ topicCreated = true;
Review comment:
Should we verify that the number of partitions in the existing topic matches the configuration?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ final NewTopic remoteLogMetadataTopicRequest = createRemoteLogMetadataTopicRequest();
+
+ // Stop if it is already initialized or closing.
+ while (!(initialized.get() || closing.get())) {
+ // There were dead locks observed when the remote log metadata topic created as part of on internal
+ // topic creation(with auto create enabled) when multiple brokers were getting started and RLMM creating
+ // the respective producer and consumer instances.
+ if (!createTopic(remoteLogMetadataTopicRequest)) {
+ // try to create the topic again if it could not be created.
+ log.info("Sleep for : {} ms before it is retried again.", INITIALIZATION_RETRY_INTERVAL_MS);
+ Utils.sleep(INITIALIZATION_RETRY_INTERVAL_MS);
Review comment:
Hmm, we don't want to retry forever. If there is a configuration error, we want to fail fast.
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerHarness.java
##########
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import kafka.api.IntegrationTestHarness;
+import kafka.utils.TestUtils;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerHarness extends IntegrationTestHarness {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerHarness.class);
+
+ protected static final int METADATA_TOPIC_PARTITIONS_COUNT = 3;
+ protected static final int METADATA_TOPIC_REPLICATION_FACTOR = 2;
+ protected static final long METADATA_TOPIC_RETENTION_MS = 24 * 60 * 60 * 1000L;
+
+ private final Time time = new MockTime(1);
+ private TopicBasedRemoteLogMetadataManager topicBasedRemoteLogMetadataManager;
+
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.emptyMap();
+ }
+
+ public void initialize(Set<TopicIdPartition> topicIdPartitions) {
+ // Call setup to start the cluster.
+ super.setUp();
+
+ // Make sure the remote log metadata topic is created before it is used.
+ createMetadataTopic();
+
+ topicBasedRemoteLogMetadataManager = new TopicBasedRemoteLogMetadataManager(time);
+
+ // Initialize TopicBasedRemoteLogMetadataManager.
+ Map<String, Object> configs = new HashMap<>();
+ configs.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList());
+ configs.put("broker.id", 0);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, METADATA_TOPIC_PARTITIONS_COUNT);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, METADATA_TOPIC_REPLICATION_FACTOR);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, METADATA_TOPIC_RETENTION_MS);
+
+ log.debug("TopicBasedRemoteLogMetadataManager configs before adding overridden properties: {}", configs);
+ // Add override properties.
+ configs.putAll(overrideRemoteLogMetadataManagerProps());
+ log.debug("TopicBasedRemoteLogMetadataManager configs after adding overridden properties: {}", configs);
+
+ topicBasedRemoteLogMetadataManager.configure(configs);
+ try {
+ waitUntilInitialized(120_000);
+ } catch (TimeoutException e) {
+ throw new RuntimeException(e);
+ }
+
+ topicBasedRemoteLogMetadataManager.onPartitionLeadershipChanges(topicIdPartitions, Collections.emptySet());
+ }
+
+ // Visible for testing.
+ public void waitUntilInitialized(long waitTimeMs) throws TimeoutException {
+ long startMs = System.currentTimeMillis();
+ while (!topicBasedRemoteLogMetadataManager.isInitialized()) {
+ long currentTimeMs = System.currentTimeMillis();
+ if (currentTimeMs > startMs + waitTimeMs) {
+ throw new TimeoutException("Time out reached before it is initialized successfully");
+ }
+
+ Utils.sleep(1000);
+ }
+ }
+
+ @Override
+ public int brokerCount() {
+ return 3;
+ }
+
+ protected TopicBasedRemoteLogMetadataManager topicBasedRlmm() {
+ return topicBasedRemoteLogMetadataManager;
+ }
+
+ private void createMetadataTopic() {
Review comment:
createMetadataTopic() is no longer used.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ final NewTopic remoteLogMetadataTopicRequest = createRemoteLogMetadataTopicRequest();
+
+ // Stop if it is already initialized or closing.
+ while (!(initialized.get() || closing.get())) {
+ // There were dead locks observed when the remote log metadata topic created as part of on internal
+ // topic creation(with auto create enabled) when multiple brokers were getting started and RLMM creating
+ // the respective producer and consumer instances.
+ if (!createTopic(remoteLogMetadataTopicRequest)) {
+ // try to create the topic again if it could not be created.
+ log.info("Sleep for : {} ms before it is retried again.", INITIALIZATION_RETRY_INTERVAL_MS);
+ Utils.sleep(INITIALIZATION_RETRY_INTERVAL_MS);
+ } else {
+ // Create producer and consumer managers.
+ lock.writeLock().lock();
+ try {
+ producerManager = new ProducerManager(rlmmConfig, rlmmTopicPartitioner);
+ consumerManager = new ConsumerManager(rlmmConfig, remotePartitionMetadataStore, rlmmTopicPartitioner, time);
+ consumerManager.startConsumerThread();
+
+ if (!pendingAssignPartitions.isEmpty()) {
+ consumerManager.addAssignmentsForPartitions(pendingAssignPartitions);
+ pendingAssignPartitions.clear();
+ }
+
+ initialized.set(true);
+ log.info("Initialized resources successfully.");
+ } catch (Exception e) {
+ log.error("Encountered error while initializing producer/consumer", e);
+ return;
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+ }
+ }
+
+ private NewTopic createRemoteLogMetadataTopicRequest() {
+ Map<String, String> topicConfigs = new HashMap<>();
+ topicConfigs.put(TopicConfig.RETENTION_MS_CONFIG, Long.toString(rlmmConfig.metadataTopicRetentionMs()));
+ topicConfigs.put(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
+ return new NewTopic(rlmmConfig.remoteLogMetadataTopicName(),
+ rlmmConfig.metadataTopicPartitionsCount(),
+ rlmmConfig.metadataTopicReplicationFactor()).configs(topicConfigs);
+ }
+
+ /**
+ * @param topic topic to be created.
+ * @return Returns true if the topic already exists or it is created successfully.
+ */
+ private boolean createTopic(NewTopic topic) {
+ boolean topicCreated = false;
+ AdminClient adminClient = null;
+ try {
+ adminClient = AdminClient.create(rlmmConfig.consumerProperties());
Review comment:
It seems producer credentials are closer for the admin client.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r649387744
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
Review comment:
We do not want this to be completely blocked as we want to release the remote log thread after a specific timeout in case of any intermittent issues so that other partitions tiring can proceed.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
+ commonClientConfigs.put(key.substring(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_PRODUCER_PREFIX)) {
+ producerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_PRODUCER_PREFIX.length()), entry.getValue());
+ } else if (key.startsWith(REMOTE_LOG_METADATA_CONSUMER_PREFIX)) {
+ consumerOnlyConfigs.put(key.substring(REMOTE_LOG_METADATA_CONSUMER_PREFIX.length()), entry.getValue());
+ }
+ }
+
+ HashMap<String, Object> allProducerConfigs = new HashMap<>(commonClientConfigs);
+ allProducerConfigs.putAll(producerOnlyConfigs);
+ producerProps = createProducerProps(allProducerConfigs);
+
+ HashMap<String, Object> allConsumerConfigs = new HashMap<>(commonClientConfigs);
+ allConsumerConfigs.putAll(consumerOnlyConfigs);
+ consumerProps = createConsumerProps(allConsumerConfigs);
+ }
+
+ public String remoteLogMetadataTopicName() {
+ return REMOTE_LOG_METADATA_TOPIC_NAME;
+ }
+
+ public int metadataTopicPartitionsCount() {
+ return metadataTopicPartitionsCount;
+ }
+
+ public long consumeWaitMs() {
+ return consumeWaitMs;
+ }
+
+ public Map<String, Object> consumerProperties() {
+ return consumerProps;
+ }
+
+ public Map<String, Object> producerProperties() {
+ return producerProps;
+ }
+
+ private Map<String, Object> createConsumerProps(HashMap<String, Object> allConsumerConfigs) {
+ Map<String, Object> props = new HashMap<>(allConsumerConfigs);
+
+ props.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientIdPrefix + "_consumer");
+ props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
+ props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ props.put(ConsumerConfig.EXCLUDE_INTERNAL_TOPICS_CONFIG, false);
+ props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
+ props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
+ return props;
+ }
+
+ private Map<String, Object> createProducerProps(HashMap<String, Object> allProducerConfigs) {
+ Map<String, Object> props = new HashMap<>(allProducerConfigs);
+
+ props.put(ProducerConfig.CLIENT_ID_CONFIG, clientIdPrefix + "_producer");
+ props.put(ProducerConfig.ACKS_CONFIG, "all");
+ props.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, 1);
+ props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
+ props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
+
+ return Collections.unmodifiableMap(props);
+ }
+
+ @Override
+ public String toString() {
+ return "TopicBasedRemoteLogMetadataManagerConfig{" +
+ "clientIdPrefix='" + clientIdPrefix + '\'' +
+ ", metadataTopicPartitionsCount=" + metadataTopicPartitionsCount +
+ ", bootstrapServers='" + bootstrapServers + '\'' +
+ ", consumeWaitMs=" + consumeWaitMs +
+ ", metadataTopicRetentionMillis=" + metadataTopicRetentionMillis +
+ ", consumerProps=" + consumerProps +
Review comment:
We are using HashMap for these instances and it prints k,v format. Are you suggesting that this map implementation may change as it is of type Map and need to put the right toString. We can change the reference type to HashMap for clarity if needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao merged pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao merged pull request #10579:
URL: https://github.com/apache/kafka/pull/10579
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663809083
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogSegmentLifecycleManager.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Optional;
+
+/**
+ * This interface defines the lifecycle methods for {@code RemoteLogSegmentMetadata}. {@link RemoteLogSegmentLifecycleTest} tests
+ * different implementations of this interface. This is responsible for managing all the segments for a given {@code topicIdPartition}
+ * registered with {@link #initialize(TopicIdPartition)}.
+ */
+public interface RemoteLogSegmentLifecycleManager extends Closeable {
Review comment:
There are two implementations about this class for both `RemoteLogMetadataCache` and `TopicBasedRemoteLogMetadataManager`. There are common tests in `RemoteLogSegmentLifecycleTest `that we want to run for both of them.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r666697373
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
@junrao
As I mentioned earlier, RLMM is created before broker starts accepting the requests. So, when RLMM is getting initialized and tries to create a topic in the same thread, then none of the brokers(including the brokers related to the bootstrap-servers config) will be available for taking any admin client requests for topic creation. That is why I was doing this in a different thread as it allows the broker/controller to comeup and accept the admin client request for topic creation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r642095217
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ while (true) {
+ if (!initialized.get() && !close.get()) {
+ try {
+ //todo: There were race conditions observed in creating the remote log metadata topic when multiple
+ // brokers started running RLMM and creating the topic based on auto create flag.
+ // We may want to add a check here later.
+
+ // Create producer and consumer managers.
+ if (producerManager == null) {
+ producerManager = new ProducerManager(rlmmConfig, rlmmTopicPartitioner);
+ }
+
+ if (consumerManager == null) {
+ consumerManager = new ConsumerManager(rlmmConfig, remotePartitionMetadataStore, rlmmTopicPartitioner, time);
+ consumerManager.startConsumerThread();
+ }
+
+ initialized.set(true);
+ log.info("Initialized resources successfully.");
+ break;
+ } catch (Exception e) {
+ log.error("Encountered error while initializing producer/consumer", e);
+
+ // If it is already closing, break from here before it retries.
+ if (close.get()) {
+ break;
+ }
+
+ log.error("Sleep for : {} ms before it is retried again.", INITIALIZATION_RETRY_INTERVAL_MS);
+ Utils.sleep(INITIALIZATION_RETRY_INTERVAL_MS);
+ }
+ }
+ }
+ }
+
+ public boolean isInitialized() {
+ return initialized.get();
+ }
+
+ private void ensureInitializedAndNotClosed() {
+ if (close.get() || !initialized.get()) {
+ throw new IllegalStateException("This instance is in invalid state, initialized: " + initialized +
+ " close: " + close);
+ }
+ }
+
+ // Visible for testing.
+ void setTime(Time time) {
Review comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao merged pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao merged pull request #10579:
URL: https://github.com/apache/kafka/pull/10579
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r671094510
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.FatalExitError;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private final Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
Review comment:
It is needed because `pendingAssignPartitions` is updated in both `onPartitionLeadershipChanges` and `onStopPartitions` methods which can happen concurrently as both of them take read lock.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r671415369
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.FatalExitError;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private final Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+ private volatile boolean initializationFailed;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
Review comment:
That is a good point. We plan to improve the semantics here.
Earlier, we plan to introduce RetriableException for RLMM and RSM so that callers can have an option to know whether they can retry or not. In the case of initialization is not yet complete, RetriableException can be thrown caller can retry based on backoff. RLMM can send non retriable exception if it is in closing state and there will not be any retries.
Another way to handle this is to take these events and store them in in-memory queue and return Future. These Futures will be considered successful if initialization is successful and the events are published to the topic.
I plan to address these in a followup PR while these APIs are integrated with RLM, filed https://issues.apache.org/jira/browse/KAFKA-13097.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-874000725
Thanks @junrao for your comments. Addressed them with the latest commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663818344
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemotePartitionMetadataStore.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteState;
+import org.apache.kafka.server.log.remote.storage.RemoteResourceNotFoundException;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RemotePartitionMetadataStore extends RemotePartitionMetadataEventHandler implements Closeable {
Review comment:
Updated with a comment.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657671527
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ if (assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ } else {
+ log.debug("This event {} is skipped as the topic partition is not assigned for this instance.", remoteLogMetadata);
+ }
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+ assignedTopicPartitions = Collections.unmodifiableSet(updatedReassignedPartitions);
+ log.debug("Assigned topic partitions: {}", assignedTopicPartitions);
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
+ assignedMetaPartitions = Collections.unmodifiableSet(updatedAssignedMetaPartitions);
+ log.debug("Assigned metadata topic partitions: {}", assignedMetaPartitions);
+ assignPartitions = true;
+ assignPartitionsLock.notifyAll();
+ } else {
+ log.debug("No change in assigned metadata topic partitions: {}", assignedMetaPartitions);
+ }
+ }
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int partition) {
+ return Optional.ofNullable(partitionToConsumedOffsets.get(partition));
+ }
+
+ public boolean assignedPartition(int partition) {
Review comment:
What about `isPartitionAssigned`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-851023730
Thanks @junrao for your comment. Addressed most of them with the latest commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r642095239
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
Review comment:
Removed the check as you suggsted.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
Review comment:
This method is invoked from multiple threads for different topic partitions. This will not be a bottleneck as each partition's segments will be uploaded in a sequential manner.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadataContext) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNo, remoteLogMetadataContext);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNo >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNo +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNo, null,
+ serde.serialize(remoteLogMetadataContext)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() != null) {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
Review comment:
It is needed as the method is declared with throws KafkaException. I am also fine with removing it as it is a RTE.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadataContext) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNo, remoteLogMetadataContext);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNo >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNo +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNo, null,
+ serde.serialize(remoteLogMetadataContext)), callback).get();
Review comment:
Multiple events are published by multiple threads and batching will occur in the producer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r665531236
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
I made the mentioned above changes in the latest commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r665259927
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
@junrao The reason why we need to initialize in a different thread here is that RLMM will be created and `configure()` will be called before the broker starts accepting the requests. So, we can not call topic creation requests in the same thread as the brokers are not yet up.
Another way to do this is to have this topic as an internal topic and it will be auto created whenever it is accessed. afaik, creating producer and consumer instances can be done without the brokers up and running and they will not trigger a request to auto creation of remote log metadata topic. Consumer assignment will send a metadata request which will trigger auto creation of topics. This assignment on consumer is done only when RLMM receives callback thorough the broker about leader and isr updates from the controller. This is what we had in pre-2.7 implementation but we saw an intermittent deadlock issue but I do not see that happening on trunk.
In the current PR, I will make the existing producer manager and consumer manager in configure() and have the topic creation done in the tests.
I will have a quick follow-up PR with internal topic changes and remove the topic creation code from tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r642095193
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ while (true) {
+ if (!initialized.get() && !close.get()) {
+ try {
+ //todo: There were race conditions observed in creating the remote log metadata topic when multiple
+ // brokers started running RLMM and creating the topic based on auto create flag.
+ // We may want to add a check here later.
+
+ // Create producer and consumer managers.
+ if (producerManager == null) {
Review comment:
Making volatile makes sense to me.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r666372073
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
@satishd : To create a topic, you don't need a particular broker to be up. You just need to be able to access the bootstrap brokers. Auto creating this topic on access is a bit weird since it's not truly an internal topic and it is just an implementation detail of RLMM. So, it makes more sense for topic-based RLMM to create it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663811753
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerHarness.java
##########
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import kafka.api.IntegrationTestHarness;
+import kafka.utils.TestUtils;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerHarness extends IntegrationTestHarness {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerHarness.class);
+
+ protected static final int METADATA_TOPIC_PARTITIONS_COUNT = 3;
+ protected static final int METADATA_TOPIC_REPLICATION_FACTOR = 2;
+ protected static final long METADATA_TOPIC_RETENTION_MS = 24 * 60 * 60 * 1000L;
+
+ private final Time time = new MockTime(1);
+ private TopicBasedRemoteLogMetadataManager topicBasedRemoteLogMetadataManager;
+
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.emptyMap();
+ }
+
+ public void initialize(Set<TopicIdPartition> topicIdPartitions) {
+ // Call setup to start the cluster.
+ super.setUp();
+
+ // Make sure the remote log metadata topic is created before it is used.
+ createMetadataTopic();
+
+ topicBasedRemoteLogMetadataManager = new TopicBasedRemoteLogMetadataManager(time);
+
+ // Initialize TopicBasedRemoteLogMetadataManager.
+ Map<String, Object> configs = new HashMap<>();
+ configs.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList());
+ configs.put("broker.id", 0);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, METADATA_TOPIC_PARTITIONS_COUNT);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, METADATA_TOPIC_REPLICATION_FACTOR);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, METADATA_TOPIC_RETENTION_MS);
+
+ log.debug("TopicBasedRemoteLogMetadataManager configs before adding overridden properties: {}", configs);
+ // Add override properties.
+ configs.putAll(overrideRemoteLogMetadataManagerProps());
+ log.debug("TopicBasedRemoteLogMetadataManager configs after adding overridden properties: {}", configs);
+
+ topicBasedRemoteLogMetadataManager.configure(configs);
+ try {
+ waitUntilInitialized(120_000);
+ } catch (TimeoutException e) {
+ throw new RuntimeException(e);
+ }
+
+ topicBasedRemoteLogMetadataManager.onPartitionLeadershipChanges(topicIdPartitions, Collections.emptySet());
+ }
+
+ // Visible for testing.
+ public void waitUntilInitialized(long waitTimeMs) throws TimeoutException {
+ long startMs = System.currentTimeMillis();
+ while (!topicBasedRemoteLogMetadataManager.isInitialized()) {
+ long currentTimeMs = System.currentTimeMillis();
+ if (currentTimeMs > startMs + waitTimeMs) {
+ throw new TimeoutException("Time out reached before it is initialized successfully");
+ }
+
+ Utils.sleep(1000);
+ }
+ }
+
+ @Override
+ public int brokerCount() {
+ return 3;
+ }
+
+ protected TopicBasedRemoteLogMetadataManager topicBasedRlmm() {
+ return topicBasedRemoteLogMetadataManager;
+ }
+
+ private void createMetadataTopic() {
Review comment:
I was planning to add that later as mentioned earlier. I updated with the required changes in latest commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] ccding commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
ccding commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664630725
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -220,16 +227,16 @@ public boolean isPartitionAssigned(int partition) {
return assignedMetaPartitions.contains(partition);
}
- private void ensureNotClosed() {
- if (closing) {
- throw new IllegalStateException("This instance is already closed");
- }
- }
-
public void close() {
if (!closing) {
- closing = true;
- consumer.wakeup();
+ synchronized (assignPartitionsLock) {
+ // Closing should be updated only after acquiring the lock to avoid race in
+ // maybeWaitForPartitionsAssignment() where it waits on assignPartitionsLock. It should not wait
+ // if the closing is already set.
+ closing = true;
Review comment:
Is it possible two threads call `close()` concurrently?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-874000725
Thanks @junrao for your comments. Addressed them with the latest commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r670947570
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.FatalExitError;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private final Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+ private volatile boolean initializationFailed;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ final NewTopic remoteLogMetadataTopicRequest = createRemoteLogMetadataTopicRequest();
+ boolean topicCreated = false;
+ long startTimeMs = time.milliseconds();
+ AdminClient adminClient = null;
+ try {
+ adminClient = AdminClient.create(rlmmConfig.producerProperties());
+
+ // Stop if it is already initialized or closing.
+ while (!(initialized.get() || closing.get())) {
+
+ // If it is timed out then raise an error to exit.
+ if (time.milliseconds() - startTimeMs > rlmmConfig.initializationRetryMaxTimeoutMs()) {
+ log.error("Timed out in initializing the resources, retried to initialize the resource for [{}] ms.",
+ rlmmConfig.initializationRetryMaxTimeoutMs());
+ initializationFailed = true;
Review comment:
Yes, we are already doing that [here](https://github.com/apache/kafka/pull/10579/files#diff-50bf08ff4e92c16dfbea9239c89467a3ebf5fa38e5f4ea44ed4acff80013136eR452) whenever this class is accessed with/for remote log metadata operations.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r671415369
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.FatalExitError;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private final Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+ private volatile boolean initializationFailed;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
Review comment:
That is a good point. We need better semantics here.
Earlier, we plan to introduce RetriableException for RLMM and RSM so that callers can have an option to know whether they can retry or not. In the case of initialization is not yet complete, RetriableException can be thrown caller can retry based on backoff. RLMM can send non retriable exception if it is in closing state and there will not be any retries.
Another way to handle is to take these events and store them in in-memory queue and return Future. These Futures will be considered successful if initialization is successful and the events are published to the topic.
I plan to address these in a followup PR while these APIs are integrated with RLM, filed https://issues.apache.org/jira/browse/KAFKA-13097.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657655966
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not. It will not accept any
+ // messages once it is set as true.
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
Review comment:
What about `must be less than the partition count`, conveys the intent clearly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664976317
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 100;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. If it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean closing = false;
+ // It indicates whether the consumer needs to assign the partitions or not. This is set when it is
+ // determined that the consumer needs to be assigned with the updated partitions.
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ Objects.requireNonNull(consumer);
+ Objects.requireNonNull(remotePartitionMetadataEventHandler);
+ Objects.requireNonNull(topicPartitioner);
+
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!closing) {
+ maybeWaitForPartitionsAssignment();
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofMillis(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", closing, e);
+ } finally {
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+
+ private void maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ // If it is closing, return immediately. This should be inside the assignPartitionsLock as the closing is updated
+ // in close() method with in the same lock to avoid any race conditions.
+ if (closing) {
+ return;
+ }
+
+ while (assignedMetaPartitions.isEmpty() && !closing) {
Review comment:
This loop can run after `assignPartitionsLock.wait()` call which might have been notified from `close()` method, we should have a `closing` check to get out of the loop.
We can have a more aggressive check to return from here when `closing` is true, I will add that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r665531236
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
I made the mentioned changes for this PR in the latest commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-843289802
@junrao @kowshik This PR is ready for review. Pl let me know your comments.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663821465
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
Review comment:
`closing` is accessed in `initializeResources` and we do not need to take a lock there. I would like to keep this as `AtomicBoolean` which addresses that and it is easy to understand the semantics.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-881556895
Thanks @junrao for your latest comments, replied to them inline.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r671367565
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
No, it is not needed to be sent with any prefix(like common_client, producer or consumer) because "bootstrap.servers" property is sent for any registered RLMM plugin but not only limited to the default RLMM. I will update the KIP with these details.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657655825
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ if (assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ } else {
+ log.debug("This event {} is skipped as the topic partition is not assigned for this instance.", remoteLogMetadata);
+ }
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
Review comment:
When the code was refactored, it went with the caller method arg names. thanks for catching these.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664970217
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogSegmentLifecycleManager.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Optional;
+
+/**
+ * This interface defines the lifecycle methods for {@code RemoteLogSegmentMetadata}. {@link RemoteLogSegmentLifecycleTest} tests
+ * different implementations of this interface. This is responsible for managing all the segments for a given {@code topicIdPartition}
+ * registered with {@link #initialize(TopicIdPartition)}.
+ */
+public interface RemoteLogSegmentLifecycleManager extends Closeable {
Review comment:
`RemoteLogSegmentLifecycleManager` is already under tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657670757
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
Review comment:
This is a temporary change. I plan to have a config with a default value.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657674359
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
Review comment:
Javadocs explain the behaviorr in detail.
`addRemoteLogSegmentMetadata` - ads a new entry.
`updateRemoteLogSegmentMetadata ` - update an existing entry.
`putRemotePartitionDeleteMetadata ` - add or update an existing entry, put is generally used for that purpose. If this is not so clear, one option can be `addOrUpdateRemotePartitionDeleteMetadata`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663813208
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerTest.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerTest {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerTest.class);
+
+ private static final int SEG_SIZE = 1024 * 1024;
+
+ private final Time time = new MockTime(1);
+ private final TopicBasedRemoteLogMetadataManagerHarness remoteLogMetadataManagerHarness = new TopicBasedRemoteLogMetadataManagerHarness() {
+ @Override
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.singletonMap(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, 5000L);
+ }
+ };
+
+ @BeforeEach
+ public void setup() {
+ // Start the cluster and initialize TopicBasedRemoteLogMetadataManager.
+ remoteLogMetadataManagerHarness.initialize(Collections.emptySet());
+ }
+
+ @AfterEach
+ public void teardown() throws IOException {
+ remoteLogMetadataManagerHarness.close();
+ }
+
+ public TopicBasedRemoteLogMetadataManager topicBasedRlmm() {
+ return remoteLogMetadataManagerHarness.topicBasedRlmm();
+ }
+
+ @Test
+ public void testNewPartitionUpdates() throws Exception {
+ final TopicIdPartition newLeaderTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("new-leader", 0));
+ final TopicIdPartition newFollowerTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("new-follower", 0));
+
+ // Add segments for these partitions but they are not available as they have not yet been subscribed.
+ RemoteLogSegmentMetadata leaderSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newLeaderTopicIdPartition, Uuid.randomUuid()),
+ 0, 100, -1L, 0,
+ time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
+ topicBasedRlmm().addRemoteLogSegmentMetadata(leaderSegmentMetadata);
+
+ RemoteLogSegmentMetadata followerSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newFollowerTopicIdPartition, Uuid.randomUuid()),
+ 0, 100, -1L, 0,
+ time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
+ topicBasedRlmm().addRemoteLogSegmentMetadata(followerSegmentMetadata);
+
+ // `listRemoteLogSegments` will receive an exception as these topic partitions are not yet registered.
+ Assertions.assertThrows(RemoteStorageException.class, () -> topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition));
+ Assertions.assertThrows(RemoteStorageException.class, () -> topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition));
+
+ topicBasedRlmm().onPartitionLeadershipChanges(Collections.singleton(newLeaderTopicIdPartition),
+ Collections.singleton(newFollowerTopicIdPartition));
+
+ waitUntilConsumerCatchesup(newLeaderTopicIdPartition, newFollowerTopicIdPartition, 30000L);
+
+ Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition).hasNext());
+ Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition).hasNext());
+ }
+
+ private void waitUntilConsumerCatchesup(TopicIdPartition newLeaderTopicIdPartition,
+ TopicIdPartition newFollowerTopicIdPartition,
+ long timeoutMs) throws TimeoutException {
+ int leaderMetadataPartition = topicBasedRlmm().metadataPartition(newLeaderTopicIdPartition);
+ int followerMetadataPartition = topicBasedRlmm().metadataPartition(newFollowerTopicIdPartition);
+
+ log.debug("Metadata partition for newLeaderTopicIdPartition: [{}], is: [{}]", newLeaderTopicIdPartition, leaderMetadataPartition);
+ log.debug("Metadata partition for newFollowerTopicIdPartition: [{}], is: [{}]", newFollowerTopicIdPartition, followerMetadataPartition);
+
+ long sleepMs = 100L;
+ long time = System.currentTimeMillis();
+
+ while (true) {
+ if (System.currentTimeMillis() - time > timeoutMs) {
+ throw new TimeoutException("Timed out after " + timeoutMs + "ms ");
+ }
+
+ if (leaderMetadataPartition == followerMetadataPartition) {
+ if (topicBasedRlmm().receivedOffsetForPartition(leaderMetadataPartition).orElse(-1L) > 0) {
+ break;
+ }
+ } else {
+ if (topicBasedRlmm().receivedOffsetForPartition(leaderMetadataPartition).orElse(-1L) > -1 ||
Review comment:
Updated with a comment in the code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r667427888
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
@junrao: That is a good point. But that is valid only for upgrade scenario.
There are two scenarios here.
1) Upgrade path
2) Fresh install/deploy with the release containing this feature.
What you said makes sense for upgrade path. But "bootstrap.servers" list is configured with the local broker endpoint only in which RLMM is getting initialized. So, if we try to initialize RLMM in the same thread, it wont be able to connect to the local broker as it is not yet started to accept the broker API requests. One way to address this is to give `bootstrap.servers` with more than one broker's endpoint. This will also put a limitation to create a cluster with one broker instance for test/demo environments.
When users install/deploy a fresh environment with the release containing this feature, there should not be a restriction to do rolling restarts with the configuration enabled. Please let me know if I am missing anything here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r671419080
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
I am not sure about usecases with other RLMM but it allows them to connect to the broker.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657656102
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not. It will not accept any
+ // messages once it is set as true.
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNum, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() == null) {
+ return callback.recordMetadata();
+ } else {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
Review comment:
If you are asking about the earlier catch block, that will not cover the exceptions like receiving from broker etc. The earlier catch block is applicable only until the record is added to the accumulator.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r671415369
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.FatalExitError;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private final Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+ private volatile boolean initializationFailed;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
Review comment:
That is a good point. We need better semantics here.
Earlier, we plan to introduce RetriableException for RLMM and RSM so that callers can have an option to know whether they can retry or not. In the case of initialization is not yet complete, RetriableException can be thrown caller can retry based on backoff. RLMM can send non retriable exception if it is in closing state and there will not be any retries.
Another way to handle here is to take these events and store them in in-memory queue and return Future. These Futures will be considered successful if initialization is successful and the events are published to the topic.
I plan to address these in a followup PR while these APIs are integrated with RLM.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-859400725
Thanks @junrao @kowshik for the review comments. Addressed them with the latest commits.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r642095431
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogMetadataTopicPartitioner.java
##########
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+public class RemoteLogMetadataTopicPartitioner {
+ public static final Logger log = LoggerFactory.getLogger(RemoteLogMetadataTopicPartitioner.class);
+ private final int noOfMetadataTopicPartitions;
+
+ public RemoteLogMetadataTopicPartitioner(int noOfMetadataTopicPartitions) {
+ this.noOfMetadataTopicPartitions = noOfMetadataTopicPartitions;
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ Objects.requireNonNull(topicIdPartition, "TopicPartition can not be null");
+
+ int partitionNo = Utils.toPositive(Utils.murmur2(topicIdPartition.toString().getBytes(StandardCharsets.UTF_8)))
Review comment:
Good point. I will add that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r642095305
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+public class ConsumerManager implements Closeable {
Review comment:
I will add that.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
Review comment:
Are you saying it should be initialized with `updatedAssignedMetaPartitions = new HashSet<>(assignedMetaPartitions);`. This is not needed as we are recomputing that set based on updatedReassignedPartitions.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
Review comment:
Nice catch. I will update the code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r660810740
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
+
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private final Time time;
+ private final ConsumerTask consumerTask;
+ private final Thread consumerTaskThread;
+
+ public ConsumerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner,
+ Time time) {
+ this.rlmmConfig = rlmmConfig;
+ this.time = time;
+
+ //Create a task to consume messages and submit the respective events to RemotePartitionMetadataEventHandler.
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(rlmmConfig.consumerProperties());
+ consumerTask = new ConsumerTask(consumer, remotePartitionMetadataEventHandler, rlmmTopicPartitioner);
+ consumerTaskThread = KafkaThread.daemon("RLMMConsumerTask", consumerTask);
+ }
+
+ public void startConsumerThread() {
+ try {
+ // Start a thread to continuously consume records from topic partitions.
+ consumerTaskThread.start();
+ } catch (Exception e) {
+ throw new KafkaException("Error encountered while initializing and scheduling ConsumerTask thread", e);
+ }
+ }
+
+ /**
+ * Wait until the consumption reaches the offset of the metadata partition for the given {@code recordMetadata}.
+ *
+ * @param recordMetadata record metadata to be checked for consumption.
+ */
+ public void waitTillConsumptionCatchesUp(RecordMetadata recordMetadata) {
+ final int partition = recordMetadata.partition();
+
+ // If the current assignment does not have the subscription for this partition then return immediately.
+ if (!consumerTask.isPartitionAssigned(partition)) {
+ log.warn("This consumer is not subscribed to the target partition [{}] on which message is produced.",
+ partition);
+ return;
Review comment:
Hmm, should we throw an exception in this case so that the caller knows the operation has failed?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
Review comment:
Since we have a lock, could we just make closing a boolean?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
Other plugins on the broker may also need a bootstrap_server config. To distinguish them, it would be useful to add a prefix that's specific to remote storage.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
+
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private final Time time;
+ private final ConsumerTask consumerTask;
+ private final Thread consumerTaskThread;
+
+ public ConsumerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner,
+ Time time) {
+ this.rlmmConfig = rlmmConfig;
+ this.time = time;
+
+ //Create a task to consume messages and submit the respective events to RemotePartitionMetadataEventHandler.
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(rlmmConfig.consumerProperties());
+ consumerTask = new ConsumerTask(consumer, remotePartitionMetadataEventHandler, rlmmTopicPartitioner);
+ consumerTaskThread = KafkaThread.daemon("RLMMConsumerTask", consumerTask);
Review comment:
Hmm, why is this a daemon thread? It seems that we want to coordinate the shutdown of the thread.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not. It will not accept any
+ // messages once it is set as true.
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
Review comment:
Is this necessary? Once producer is closed, send() will throw an exception.
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerHarness.java
##########
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import kafka.api.IntegrationTestHarness;
+import kafka.utils.TestUtils;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerHarness extends IntegrationTestHarness {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerHarness.class);
+
+ protected static final int METADATA_TOPIC_PARTITIONS_COUNT = 3;
+ protected static final int METADATA_TOPIC_REPLICATION_FACTOR = 2;
+ protected static final long METADATA_TOPIC_RETENTION_MS = 24 * 60 * 60 * 1000L;
+
+ private final Time time = new MockTime(1);
+ private TopicBasedRemoteLogMetadataManager topicBasedRemoteLogMetadataManager;
+
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.emptyMap();
+ }
+
+ public void initialize(Set<TopicIdPartition> topicIdPartitions) {
+ // Call setup to start the cluster.
+ super.setUp();
+
+ // Make sure the remote log metadata topic is created before it is used.
+ createMetadataTopic();
+
+ topicBasedRemoteLogMetadataManager = new TopicBasedRemoteLogMetadataManager(time);
+
+ // Initialize TopicBasedRemoteLogMetadataManager.
+ Map<String, Object> configs = new HashMap<>();
+ configs.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList());
+ configs.put("broker.id", 0);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, METADATA_TOPIC_PARTITIONS_COUNT);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, METADATA_TOPIC_REPLICATION_FACTOR);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, METADATA_TOPIC_RETENTION_MS);
+
+ log.debug("TopicBasedRemoteLogMetadataManager configs before adding overridden properties: {}", configs);
+ // Add override properties.
+ configs.putAll(overrideRemoteLogMetadataManagerProps());
+ log.debug("TopicBasedRemoteLogMetadataManager configs after adding overridden properties: {}", configs);
+
+ topicBasedRemoteLogMetadataManager.configure(configs);
+ try {
+ waitUntilInitialized(120_000);
+ } catch (TimeoutException e) {
+ throw new RuntimeException(e);
+ }
+
+ topicBasedRemoteLogMetadataManager.onPartitionLeadershipChanges(topicIdPartitions, Collections.emptySet());
+ }
+
+ // Visible for testing.
+ public void waitUntilInitialized(long waitTimeMs) throws TimeoutException {
+ long startMs = System.currentTimeMillis();
+ while (!topicBasedRemoteLogMetadataManager.isInitialized()) {
+ long currentTimeMs = System.currentTimeMillis();
+ if (currentTimeMs > startMs + waitTimeMs) {
+ throw new TimeoutException("Time out reached before it is initialized successfully");
+ }
+
+ Utils.sleep(1000);
+ }
+ }
+
+ @Override
+ public int brokerCount() {
+ return 3;
+ }
+
+ protected TopicBasedRemoteLogMetadataManager topicBasedRlmm() {
+ return topicBasedRemoteLogMetadataManager;
+ }
+
+ private void createMetadataTopic() {
Review comment:
It seems that we need to automatically create metadata topic in RLMM implementation, not just in tests.
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogSegmentLifecycleManager.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Optional;
+
+/**
+ * This interface defines the lifecycle methods for {@code RemoteLogSegmentMetadata}. {@link RemoteLogSegmentLifecycleTest} tests
+ * different implementations of this interface. This is responsible for managing all the segments for a given {@code topicIdPartition}
+ * registered with {@link #initialize(TopicIdPartition)}.
+ */
+public interface RemoteLogSegmentLifecycleManager extends Closeable {
Review comment:
How is this different from RemoteLogMetadataCache? It seems that it's just a wrapper over RemoteLogMetadataCache?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. If it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean closing = false;
+ // It indicates whether the consumer needs to assign the partitions or not. This is set when it is
+ // determined that the consumer needs to be assigned with the updated partitions.
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ Objects.requireNonNull(consumer);
+ Objects.requireNonNull(remotePartitionMetadataEventHandler);
+ Objects.requireNonNull(topicPartitioner);
+
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!closing) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", closing, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ if (assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ } else {
+ log.debug("This event {} is skipped as the topic partition is not assigned for this instance.", remoteLogMetadata);
+ }
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(partitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
Review comment:
Is this necessary since immediately after this check, the consumer task could be closed. Ditto in other places.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemotePartitionMetadataStore.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteState;
+import org.apache.kafka.server.log.remote.storage.RemoteResourceNotFoundException;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RemotePartitionMetadataStore extends RemotePartitionMetadataEventHandler implements Closeable {
Review comment:
Could we add a comment for this class?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. If it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean closing = false;
+ // It indicates whether the consumer needs to assign the partitions or not. This is set when it is
+ // determined that the consumer needs to be assigned with the updated partitions.
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ Objects.requireNonNull(consumer);
+ Objects.requireNonNull(remotePartitionMetadataEventHandler);
+ Objects.requireNonNull(topicPartitioner);
+
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!closing) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", closing, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
Review comment:
We need to unblock the wait if we are closing the consumer.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+/**
+ * This class defines the configuration of topic based {@link org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager} implementation.
+ */
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMs;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
Review comment:
Is this needed since we log all configs when creating KafkaConfig already?
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogSegmentLifecycleTest.java
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemoteResourceNotFoundException;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.apache.kafka.test.TestUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+public class RemoteLogSegmentLifecycleTest {
+ private static final Logger log = LoggerFactory.getLogger(RemoteLogSegmentLifecycleTest.class);
+
+ private static final int SEG_SIZE = 1024 * 1024;
+ private static final int BROKER_ID_0 = 0;
+ private static final int BROKER_ID_1 = 1;
+
+ private final TopicIdPartition topicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
+ private final Time time = new MockTime(1);
+
+ @ParameterizedTest(name = "remoteLogSegmentLifecycleManager = {0}")
+ @MethodSource("remoteLogSegmentLifecycleManagers")
+ public void testRemoteLogSegmentLifeCycle(RemoteLogSegmentLifecycleManager remoteLogSegmentLifecycleManager) throws Exception {
+ try {
+ remoteLogSegmentLifecycleManager.initialize(topicIdPartition);
+
+ // segment 0
+ // offsets: [0-100]
+ // leader epochs (0,0), (1,20), (2,80)
+ Map<Integer, Long> segment0LeaderEpochs = new HashMap<>();
+ segment0LeaderEpochs.put(0, 0L);
+ segment0LeaderEpochs.put(1, 20L);
+ segment0LeaderEpochs.put(2, 80L);
+ RemoteLogSegmentId segment0Id = new RemoteLogSegmentId(topicIdPartition, Uuid.randomUuid());
+ RemoteLogSegmentMetadata segment0Metadata = new RemoteLogSegmentMetadata(segment0Id, 0L, 100L,
+ -1L, BROKER_ID_0, time.milliseconds(), SEG_SIZE,
+ segment0LeaderEpochs);
+ remoteLogSegmentLifecycleManager.addRemoteLogSegmentMetadata(segment0Metadata);
+
+ // We should not get this as the segment is still getting copied and it is not yet considered successful until
+ // it reaches RemoteLogSegmentState.COPY_SEGMENT_FINISHED.
+ Assertions.assertFalse(remoteLogSegmentLifecycleManager.remoteLogSegmentMetadata(40, 1).isPresent());
+
+ // Check that these leader epochs are to considered for highest offsets as they are still getting copied and
+ // they did nto reach COPY_SEGMENT_FINISHED state.
Review comment:
typo nto
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerHarness.java
##########
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import kafka.api.IntegrationTestHarness;
+import kafka.utils.TestUtils;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerHarness extends IntegrationTestHarness {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerHarness.class);
+
+ protected static final int METADATA_TOPIC_PARTITIONS_COUNT = 3;
+ protected static final int METADATA_TOPIC_REPLICATION_FACTOR = 2;
+ protected static final long METADATA_TOPIC_RETENTION_MS = 24 * 60 * 60 * 1000L;
+
+ private final Time time = new MockTime(1);
+ private TopicBasedRemoteLogMetadataManager topicBasedRemoteLogMetadataManager;
+
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.emptyMap();
+ }
+
+ public void initialize(Set<TopicIdPartition> topicIdPartitions) {
+ // Call setup to start the cluster.
+ super.setUp();
+
+ // Make sure the remote log metadata topic is created before it is used.
+ createMetadataTopic();
+
+ topicBasedRemoteLogMetadataManager = new TopicBasedRemoteLogMetadataManager(time);
+
+ // Initialize TopicBasedRemoteLogMetadataManager.
+ Map<String, Object> configs = new HashMap<>();
+ configs.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList());
+ configs.put("broker.id", 0);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, METADATA_TOPIC_PARTITIONS_COUNT);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, METADATA_TOPIC_REPLICATION_FACTOR);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, METADATA_TOPIC_RETENTION_MS);
+
+ log.debug("TopicBasedRemoteLogMetadataManager configs before adding overridden properties: {}", configs);
+ // Add override properties.
+ configs.putAll(overrideRemoteLogMetadataManagerProps());
+ log.debug("TopicBasedRemoteLogMetadataManager configs after adding overridden properties: {}", configs);
+
+ topicBasedRemoteLogMetadataManager.configure(configs);
+ try {
+ waitUntilInitialized(120_000);
Review comment:
120s seems quite long. Do we need to wait that long?
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfigTest.java
##########
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.AbstractMap;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_CONSUMER_PREFIX;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_PRODUCER_PREFIX;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerConfigTest {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfigTest.class);
+
+ private static final String BOOTSTRAP_SERVERS = "localhost:9091";
+
+ @Test
+ public void testEmptyConfig() throws Exception {
+ // "bootstrap.servers" config is required, it will throw IllegalArgumentException if it does not exist.
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new TopicBasedRemoteLogMetadataManagerConfig(Collections.emptyMap()));
+ }
+
+ @Test
+ public void testValidConfig() {
+
+ Map<String, Object> commonClientConfig = new HashMap<>();
+ commonClientConfig.put(CommonClientConfigs.RETRIES_CONFIG, 10);
+ commonClientConfig.put(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG, 1000L);
+ commonClientConfig.put(CommonClientConfigs.METADATA_MAX_AGE_CONFIG, 60000L);
+
+ Map<String, Object> producerConfig = new HashMap<>();
+ producerConfig.put(ProducerConfig.ACKS_CONFIG, "all");
+
+ Map<String, Object> consumerConfig = new HashMap<>();
+ consumerConfig.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
+
+ Map<String, Object> props = createValidConfigProps(commonClientConfig, producerConfig, consumerConfig);
+
+ // Check for topic properties
+ TopicBasedRemoteLogMetadataManagerConfig rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(props);
+ Assertions.assertEquals(props.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP), rlmmConfig.metadataTopicPartitionsCount());
+
+ // Check for common client configs.
+ for (Map.Entry<String, Object> entry : commonClientConfig.entrySet()) {
+ log.info("Checking config: " + entry.getKey());
+ Assertions.assertEquals(entry.getValue(),
+ rlmmConfig.producerProperties().get(entry.getKey()));
+ Assertions.assertEquals(entry.getValue(),
+ rlmmConfig.producerProperties().get(entry.getKey()));
Review comment:
This should be for consumer?
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogSegmentLifecycleTest.java
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemoteResourceNotFoundException;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.apache.kafka.test.TestUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+public class RemoteLogSegmentLifecycleTest {
+ private static final Logger log = LoggerFactory.getLogger(RemoteLogSegmentLifecycleTest.class);
+
+ private static final int SEG_SIZE = 1024 * 1024;
+ private static final int BROKER_ID_0 = 0;
+ private static final int BROKER_ID_1 = 1;
+
+ private final TopicIdPartition topicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0));
+ private final Time time = new MockTime(1);
+
+ @ParameterizedTest(name = "remoteLogSegmentLifecycleManager = {0}")
+ @MethodSource("remoteLogSegmentLifecycleManagers")
+ public void testRemoteLogSegmentLifeCycle(RemoteLogSegmentLifecycleManager remoteLogSegmentLifecycleManager) throws Exception {
+ try {
+ remoteLogSegmentLifecycleManager.initialize(topicIdPartition);
+
+ // segment 0
+ // offsets: [0-100]
+ // leader epochs (0,0), (1,20), (2,80)
+ Map<Integer, Long> segment0LeaderEpochs = new HashMap<>();
+ segment0LeaderEpochs.put(0, 0L);
+ segment0LeaderEpochs.put(1, 20L);
+ segment0LeaderEpochs.put(2, 80L);
+ RemoteLogSegmentId segment0Id = new RemoteLogSegmentId(topicIdPartition, Uuid.randomUuid());
+ RemoteLogSegmentMetadata segment0Metadata = new RemoteLogSegmentMetadata(segment0Id, 0L, 100L,
+ -1L, BROKER_ID_0, time.milliseconds(), SEG_SIZE,
+ segment0LeaderEpochs);
+ remoteLogSegmentLifecycleManager.addRemoteLogSegmentMetadata(segment0Metadata);
+
+ // We should not get this as the segment is still getting copied and it is not yet considered successful until
+ // it reaches RemoteLogSegmentState.COPY_SEGMENT_FINISHED.
+ Assertions.assertFalse(remoteLogSegmentLifecycleManager.remoteLogSegmentMetadata(40, 1).isPresent());
+
+ // Check that these leader epochs are to considered for highest offsets as they are still getting copied and
Review comment:
This sentence doesn't read well.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+/**
+ * This class defines the configuration of topic based {@link org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager} implementation.
+ */
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
Review comment:
MILLIS => MS to be consistent with other places. Ditto in a few other places.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
Why do we need to initialize in a separate thread?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
+
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private final Time time;
+ private final ConsumerTask consumerTask;
+ private final Thread consumerTaskThread;
+
+ public ConsumerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner,
+ Time time) {
+ this.rlmmConfig = rlmmConfig;
+ this.time = time;
+
+ //Create a task to consume messages and submit the respective events to RemotePartitionMetadataEventHandler.
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(rlmmConfig.consumerProperties());
+ consumerTask = new ConsumerTask(consumer, remotePartitionMetadataEventHandler, rlmmTopicPartitioner);
+ consumerTaskThread = KafkaThread.daemon("RLMMConsumerTask", consumerTask);
+ }
+
+ public void startConsumerThread() {
+ try {
+ // Start a thread to continuously consume records from topic partitions.
+ consumerTaskThread.start();
+ } catch (Exception e) {
+ throw new KafkaException("Error encountered while initializing and scheduling ConsumerTask thread", e);
+ }
+ }
+
+ /**
+ * Wait until the consumption reaches the offset of the metadata partition for the given {@code recordMetadata}.
+ *
+ * @param recordMetadata record metadata to be checked for consumption.
+ */
+ public void waitTillConsumptionCatchesUp(RecordMetadata recordMetadata) {
+ final int partition = recordMetadata.partition();
+
+ // If the current assignment does not have the subscription for this partition then return immediately.
+ if (!consumerTask.isPartitionAssigned(partition)) {
+ log.warn("This consumer is not subscribed to the target partition [{}] on which message is produced.",
+ partition);
+ return;
+ }
+
+ final long offset = recordMetadata.offset();
+ long startTimeMs = time.milliseconds();
+ while (true) {
+ long committedOffset = consumerTask.receivedOffsetForPartition(partition).orElse(-1L);
+ if (committedOffset >= offset) {
+ break;
+ }
+
+ log.debug("Committed offset [{}] for partition [{}], but the target offset: [{}], Sleeping for [{}] to retry again",
+ offset, partition, committedOffset, CONSUME_RECHECK_INTERVAL_MS);
+
+ if (time.milliseconds() - startTimeMs > rlmmConfig.consumeWaitMs()) {
+ log.warn("Committed offset for partition:[{}] is : [{}], but the target offset: [{}] ",
Review comment:
Similar to the above, should we throw an exception in this case so that the caller knows the operation has failed?
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerTest.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerTest {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerTest.class);
+
+ private static final int SEG_SIZE = 1024 * 1024;
+
+ private final Time time = new MockTime(1);
+ private final TopicBasedRemoteLogMetadataManagerHarness remoteLogMetadataManagerHarness = new TopicBasedRemoteLogMetadataManagerHarness() {
+ @Override
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.singletonMap(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, 5000L);
+ }
+ };
+
+ @BeforeEach
+ public void setup() {
+ // Start the cluster and initialize TopicBasedRemoteLogMetadataManager.
+ remoteLogMetadataManagerHarness.initialize(Collections.emptySet());
+ }
+
+ @AfterEach
+ public void teardown() throws IOException {
+ remoteLogMetadataManagerHarness.close();
+ }
+
+ public TopicBasedRemoteLogMetadataManager topicBasedRlmm() {
+ return remoteLogMetadataManagerHarness.topicBasedRlmm();
+ }
+
+ @Test
+ public void testNewPartitionUpdates() throws Exception {
+ final TopicIdPartition newLeaderTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("new-leader", 0));
+ final TopicIdPartition newFollowerTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("new-follower", 0));
+
+ // Add segments for these partitions but they are not available as they have not yet been subscribed.
+ RemoteLogSegmentMetadata leaderSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newLeaderTopicIdPartition, Uuid.randomUuid()),
+ 0, 100, -1L, 0,
+ time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
+ topicBasedRlmm().addRemoteLogSegmentMetadata(leaderSegmentMetadata);
+
+ RemoteLogSegmentMetadata followerSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newFollowerTopicIdPartition, Uuid.randomUuid()),
+ 0, 100, -1L, 0,
+ time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
+ topicBasedRlmm().addRemoteLogSegmentMetadata(followerSegmentMetadata);
+
+ // `listRemoteLogSegments` will receive an exception as these topic partitions are not yet registered.
+ Assertions.assertThrows(RemoteStorageException.class, () -> topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition));
+ Assertions.assertThrows(RemoteStorageException.class, () -> topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition));
+
+ topicBasedRlmm().onPartitionLeadershipChanges(Collections.singleton(newLeaderTopicIdPartition),
+ Collections.singleton(newFollowerTopicIdPartition));
+
+ waitUntilConsumerCatchesup(newLeaderTopicIdPartition, newFollowerTopicIdPartition, 30000L);
+
+ Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition).hasNext());
+ Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition).hasNext());
+ }
+
+ private void waitUntilConsumerCatchesup(TopicIdPartition newLeaderTopicIdPartition,
+ TopicIdPartition newFollowerTopicIdPartition,
+ long timeoutMs) throws TimeoutException {
+ int leaderMetadataPartition = topicBasedRlmm().metadataPartition(newLeaderTopicIdPartition);
+ int followerMetadataPartition = topicBasedRlmm().metadataPartition(newFollowerTopicIdPartition);
+
+ log.debug("Metadata partition for newLeaderTopicIdPartition: [{}], is: [{}]", newLeaderTopicIdPartition, leaderMetadataPartition);
+ log.debug("Metadata partition for newFollowerTopicIdPartition: [{}], is: [{}]", newFollowerTopicIdPartition, followerMetadataPartition);
+
+ long sleepMs = 100L;
+ long time = System.currentTimeMillis();
+
+ while (true) {
+ if (System.currentTimeMillis() - time > timeoutMs) {
+ throw new TimeoutException("Timed out after " + timeoutMs + "ms ");
+ }
+
+ if (leaderMetadataPartition == followerMetadataPartition) {
+ if (topicBasedRlmm().receivedOffsetForPartition(leaderMetadataPartition).orElse(-1L) > 0) {
+ break;
+ }
+ } else {
+ if (topicBasedRlmm().receivedOffsetForPartition(leaderMetadataPartition).orElse(-1L) > -1 ||
Review comment:
Why are we testing against -1 here but 0 above?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-875750234
Thanks @junrao for the comments. Pl take a look at my inline replies and the latest commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664983511
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -51,6 +58,7 @@
*/
public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
Review comment:
I prefer adding L for longs, which I missed at other declaration. afaik, that does not cause any issues as it gets automatically converted via a widening conversion to a `long`. The compiler takes care of not allowing numbers that may get truncated from `int` to `long` widening. Thanks for catching it, I will make it consistent by adding it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664014932
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
Review comment:
There can be a race here between this method and initializeResources() which I plan to address in a followup PR. I am working on a separate consumer to act on assigned partitions to cathcup. This race is being handled with those changes., same applies to onStopPartitions method too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r649387317
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
Review comment:
I plan to add this once RLMM is called from remote log layer classes. I wanted this change to be self contained for now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663808257
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
+
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private final Time time;
+ private final ConsumerTask consumerTask;
+ private final Thread consumerTaskThread;
+
+ public ConsumerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner,
+ Time time) {
+ this.rlmmConfig = rlmmConfig;
+ this.time = time;
+
+ //Create a task to consume messages and submit the respective events to RemotePartitionMetadataEventHandler.
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(rlmmConfig.consumerProperties());
+ consumerTask = new ConsumerTask(consumer, remotePartitionMetadataEventHandler, rlmmTopicPartitioner);
+ consumerTaskThread = KafkaThread.daemon("RLMMConsumerTask", consumerTask);
Review comment:
Good catch, updated it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663808257
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
+
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private final Time time;
+ private final ConsumerTask consumerTask;
+ private final Thread consumerTaskThread;
+
+ public ConsumerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner,
+ Time time) {
+ this.rlmmConfig = rlmmConfig;
+ this.time = time;
+
+ //Create a task to consume messages and submit the respective events to RemotePartitionMetadataEventHandler.
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(rlmmConfig.consumerProperties());
+ consumerTask = new ConsumerTask(consumer, remotePartitionMetadataEventHandler, rlmmTopicPartitioner);
+ consumerTaskThread = KafkaThread.daemon("RLMMConsumerTask", consumerTask);
Review comment:
Good catch, updated it.
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogSegmentLifecycleManager.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Optional;
+
+/**
+ * This interface defines the lifecycle methods for {@code RemoteLogSegmentMetadata}. {@link RemoteLogSegmentLifecycleTest} tests
+ * different implementations of this interface. This is responsible for managing all the segments for a given {@code topicIdPartition}
+ * registered with {@link #initialize(TopicIdPartition)}.
+ */
+public interface RemoteLogSegmentLifecycleManager extends Closeable {
Review comment:
There are two implementations about this class for both `RemoteLogMetadataCache` and `TopicBasedRemoteLogMetadataManager`. There are common tests in `RemoteLogSegmentLifecycleTest `that we want to run for both of them.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
It was required to be retried until the topic is successfully created. I added the logic to check for topic creation too.
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerHarness.java
##########
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import kafka.api.IntegrationTestHarness;
+import kafka.utils.TestUtils;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerHarness extends IntegrationTestHarness {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerHarness.class);
+
+ protected static final int METADATA_TOPIC_PARTITIONS_COUNT = 3;
+ protected static final int METADATA_TOPIC_REPLICATION_FACTOR = 2;
+ protected static final long METADATA_TOPIC_RETENTION_MS = 24 * 60 * 60 * 1000L;
+
+ private final Time time = new MockTime(1);
+ private TopicBasedRemoteLogMetadataManager topicBasedRemoteLogMetadataManager;
+
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.emptyMap();
+ }
+
+ public void initialize(Set<TopicIdPartition> topicIdPartitions) {
+ // Call setup to start the cluster.
+ super.setUp();
+
+ // Make sure the remote log metadata topic is created before it is used.
+ createMetadataTopic();
+
+ topicBasedRemoteLogMetadataManager = new TopicBasedRemoteLogMetadataManager(time);
+
+ // Initialize TopicBasedRemoteLogMetadataManager.
+ Map<String, Object> configs = new HashMap<>();
+ configs.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList());
+ configs.put("broker.id", 0);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, METADATA_TOPIC_PARTITIONS_COUNT);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, METADATA_TOPIC_REPLICATION_FACTOR);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, METADATA_TOPIC_RETENTION_MS);
+
+ log.debug("TopicBasedRemoteLogMetadataManager configs before adding overridden properties: {}", configs);
+ // Add override properties.
+ configs.putAll(overrideRemoteLogMetadataManagerProps());
+ log.debug("TopicBasedRemoteLogMetadataManager configs after adding overridden properties: {}", configs);
+
+ topicBasedRemoteLogMetadataManager.configure(configs);
+ try {
+ waitUntilInitialized(120_000);
+ } catch (TimeoutException e) {
+ throw new RuntimeException(e);
+ }
+
+ topicBasedRemoteLogMetadataManager.onPartitionLeadershipChanges(topicIdPartitions, Collections.emptySet());
+ }
+
+ // Visible for testing.
+ public void waitUntilInitialized(long waitTimeMs) throws TimeoutException {
+ long startMs = System.currentTimeMillis();
+ while (!topicBasedRemoteLogMetadataManager.isInitialized()) {
+ long currentTimeMs = System.currentTimeMillis();
+ if (currentTimeMs > startMs + waitTimeMs) {
+ throw new TimeoutException("Time out reached before it is initialized successfully");
+ }
+
+ Utils.sleep(1000);
+ }
+ }
+
+ @Override
+ public int brokerCount() {
+ return 3;
+ }
+
+ protected TopicBasedRemoteLogMetadataManager topicBasedRlmm() {
+ return topicBasedRemoteLogMetadataManager;
+ }
+
+ private void createMetadataTopic() {
Review comment:
I was planning to add that later as mentioned earlier. I updated with the required changes in latest commit.
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerTest.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.common.Uuid;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerTest {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerTest.class);
+
+ private static final int SEG_SIZE = 1024 * 1024;
+
+ private final Time time = new MockTime(1);
+ private final TopicBasedRemoteLogMetadataManagerHarness remoteLogMetadataManagerHarness = new TopicBasedRemoteLogMetadataManagerHarness() {
+ @Override
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.singletonMap(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, 5000L);
+ }
+ };
+
+ @BeforeEach
+ public void setup() {
+ // Start the cluster and initialize TopicBasedRemoteLogMetadataManager.
+ remoteLogMetadataManagerHarness.initialize(Collections.emptySet());
+ }
+
+ @AfterEach
+ public void teardown() throws IOException {
+ remoteLogMetadataManagerHarness.close();
+ }
+
+ public TopicBasedRemoteLogMetadataManager topicBasedRlmm() {
+ return remoteLogMetadataManagerHarness.topicBasedRlmm();
+ }
+
+ @Test
+ public void testNewPartitionUpdates() throws Exception {
+ final TopicIdPartition newLeaderTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("new-leader", 0));
+ final TopicIdPartition newFollowerTopicIdPartition = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("new-follower", 0));
+
+ // Add segments for these partitions but they are not available as they have not yet been subscribed.
+ RemoteLogSegmentMetadata leaderSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newLeaderTopicIdPartition, Uuid.randomUuid()),
+ 0, 100, -1L, 0,
+ time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
+ topicBasedRlmm().addRemoteLogSegmentMetadata(leaderSegmentMetadata);
+
+ RemoteLogSegmentMetadata followerSegmentMetadata = new RemoteLogSegmentMetadata(new RemoteLogSegmentId(newFollowerTopicIdPartition, Uuid.randomUuid()),
+ 0, 100, -1L, 0,
+ time.milliseconds(), SEG_SIZE, Collections.singletonMap(0, 0L));
+ topicBasedRlmm().addRemoteLogSegmentMetadata(followerSegmentMetadata);
+
+ // `listRemoteLogSegments` will receive an exception as these topic partitions are not yet registered.
+ Assertions.assertThrows(RemoteStorageException.class, () -> topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition));
+ Assertions.assertThrows(RemoteStorageException.class, () -> topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition));
+
+ topicBasedRlmm().onPartitionLeadershipChanges(Collections.singleton(newLeaderTopicIdPartition),
+ Collections.singleton(newFollowerTopicIdPartition));
+
+ waitUntilConsumerCatchesup(newLeaderTopicIdPartition, newFollowerTopicIdPartition, 30000L);
+
+ Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newLeaderTopicIdPartition).hasNext());
+ Assertions.assertTrue(topicBasedRlmm().listRemoteLogSegments(newFollowerTopicIdPartition).hasNext());
+ }
+
+ private void waitUntilConsumerCatchesup(TopicIdPartition newLeaderTopicIdPartition,
+ TopicIdPartition newFollowerTopicIdPartition,
+ long timeoutMs) throws TimeoutException {
+ int leaderMetadataPartition = topicBasedRlmm().metadataPartition(newLeaderTopicIdPartition);
+ int followerMetadataPartition = topicBasedRlmm().metadataPartition(newFollowerTopicIdPartition);
+
+ log.debug("Metadata partition for newLeaderTopicIdPartition: [{}], is: [{}]", newLeaderTopicIdPartition, leaderMetadataPartition);
+ log.debug("Metadata partition for newFollowerTopicIdPartition: [{}], is: [{}]", newFollowerTopicIdPartition, followerMetadataPartition);
+
+ long sleepMs = 100L;
+ long time = System.currentTimeMillis();
+
+ while (true) {
+ if (System.currentTimeMillis() - time > timeoutMs) {
+ throw new TimeoutException("Timed out after " + timeoutMs + "ms ");
+ }
+
+ if (leaderMetadataPartition == followerMetadataPartition) {
+ if (topicBasedRlmm().receivedOffsetForPartition(leaderMetadataPartition).orElse(-1L) > 0) {
+ break;
+ }
+ } else {
+ if (topicBasedRlmm().receivedOffsetForPartition(leaderMetadataPartition).orElse(-1L) > -1 ||
Review comment:
Updated with a comment in the code.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemotePartitionMetadataStore.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteState;
+import org.apache.kafka.server.log.remote.storage.RemoteResourceNotFoundException;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RemotePartitionMetadataStore extends RemotePartitionMetadataEventHandler implements Closeable {
Review comment:
Updated with a comment.
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerHarness.java
##########
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import kafka.api.IntegrationTestHarness;
+import kafka.utils.TestUtils;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerHarness extends IntegrationTestHarness {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerHarness.class);
+
+ protected static final int METADATA_TOPIC_PARTITIONS_COUNT = 3;
+ protected static final int METADATA_TOPIC_REPLICATION_FACTOR = 2;
+ protected static final long METADATA_TOPIC_RETENTION_MS = 24 * 60 * 60 * 1000L;
+
+ private final Time time = new MockTime(1);
+ private TopicBasedRemoteLogMetadataManager topicBasedRemoteLogMetadataManager;
+
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.emptyMap();
+ }
+
+ public void initialize(Set<TopicIdPartition> topicIdPartitions) {
+ // Call setup to start the cluster.
+ super.setUp();
+
+ // Make sure the remote log metadata topic is created before it is used.
+ createMetadataTopic();
+
+ topicBasedRemoteLogMetadataManager = new TopicBasedRemoteLogMetadataManager(time);
+
+ // Initialize TopicBasedRemoteLogMetadataManager.
+ Map<String, Object> configs = new HashMap<>();
+ configs.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList());
+ configs.put("broker.id", 0);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, METADATA_TOPIC_PARTITIONS_COUNT);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, METADATA_TOPIC_REPLICATION_FACTOR);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, METADATA_TOPIC_RETENTION_MS);
+
+ log.debug("TopicBasedRemoteLogMetadataManager configs before adding overridden properties: {}", configs);
+ // Add override properties.
+ configs.putAll(overrideRemoteLogMetadataManagerProps());
+ log.debug("TopicBasedRemoteLogMetadataManager configs after adding overridden properties: {}", configs);
+
+ topicBasedRemoteLogMetadataManager.configure(configs);
+ try {
+ waitUntilInitialized(120_000);
Review comment:
I guess 60s may be sufficient, updated with that.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
Review comment:
`closing` is accessed in `initializeResources` and we do not need to take a lock there. I would like to keep this as `AtomicBoolean` which addresses that and it is easy to understand the semantics.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
Review comment:
There can be a race here between this method and initializeResources() which I plan to address in a followup PR. I am working on a separate consumer to act on assigned partitions to cathcup. This race is being handled with those changes., same applies to onStopPartitions method too.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
Review comment:
There can be a race here between this method and initializeResources() which I plan to address in a followup PR. I am working on a separate consumer to act on assigned partitions to cathcup. This race is being handled with those changes., same applies to onStopPartitions method too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r658361692
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
Review comment:
I plan to have a config with a default value in a follow-up PR.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r671383087
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
Hmm, "bootstrap.servers" makes sense for a topic based RLMM since it depends on Kafka. Why do we require "bootstrap.servers" in other RLMM implementations?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.admin.TopicDescription;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.internals.FatalExitError;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private final Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+ private volatile boolean initializationFailed;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
Review comment:
Since producerManager is initialized asynchronously, how do we deal with the case when the producerManager is not ready when an event needs to be published?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r642095137
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ while (true) {
Review comment:
That makes sense to me.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657674359
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
Review comment:
Javadocs explain the behaviorr in detail.
`addRemoteLogSegmentMetadata` - ads a new entry.
`updateRemoteLogSegmentMetadata ` - update an existing entry.
`putRemotePartitionDeleteMetadata ` - add or update an existing entry.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r667102624
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
@satishd : My understanding is that when we enable remote storage, we will do that through a rolling upgrade. So, at any given time, there is at most a single broker being down. Therefore, as long as the bootstrap broker list contains more than one broker, operations like creating topics can still be done while a single broker is starting.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r650456266
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
Review comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r649390143
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNo, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNo >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNo +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNo, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() != null) {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
+ } else {
+ throw new KafkaException(ex);
+ }
+ } else {
+ return callback.recordMetadata();
+ }
+ }
+
+ private void ensureNotClosed() {
+ if (close) {
+ throw new IllegalStateException("This instance is already set in close state.");
Review comment:
Done
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
Review comment:
Done
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
Review comment:
I plan to add that later.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
Review comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r649390677
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
Review comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657683814
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
Review comment:
As we discussed offlime, this may not become a bottleneck but we will make respective RLMM APIs asynchronous so that the APIs are extensible and implementors can provide async behavior.FIled [KAFKA-12988](https://issues.apache.org/jira/browse/KAFKA-12988) to track this issue.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r642095314
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
Review comment:
Nice catch, updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-881805037
Thanks @junrao for the comment. I addressed it with the latest commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r667427888
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+ private volatile Set<TopicIdPartition> pendingAssignPartitions = Collections.synchronizedSet(new HashSet<>());
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then keep them as pending partitions and assign them
+ // when it is initialized successfully in initializeResources().
+ this.pendingAssignPartitions.addAll(allPartitions);
+ } else {
+ this.pendingAssignPartitions.clear();
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ if (closing.get()) {
+ throw new IllegalStateException("This instance is in closing state");
+ }
+
+ if (!initialized.get()) {
+ // If it is not yet initialized, then remove them from the pending partitions if any.
+ if (!pendingAssignPartitions.isEmpty()) {
+ pendingAssignPartitions.removeAll(partitions);
+ }
+ } else {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ }
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.nonDaemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
That is a good point. But that is valid only for upgrade scenario.
There are two scenarios here.
1) Upgrade path
2) Fresh install/deploy with the release containing this feature.
What you said makes sense for upgrade path. But "bootstrap.servers" list is configured with the local broker endpoint only in which RLMM is getting initialized. So, if we try to initialize RLMM in the same thread, it wont be able to connect to the local broker as it is not yet started to accept the broker API requests. One way to address this is to give bootstrap.servers with more than one broker's endpoint. This will also put a limiation to create a cluster with one broker instance for test/demo environments.
When users install/deploy a fresh environment with the relase contianing this feature, there should not be a restriction to do rolling restarts with the configuration enabled. Please let me know if I am missing anything here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] junrao commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
junrao commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r640154886
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadataContext) throws KafkaException {
Review comment:
remoteLogMetadataContext => remoteLogMetadata?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ while (true) {
Review comment:
If the producer/consumer are configured incorrectly, we want to fail faster instead of retrying.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+public class ConsumerManager implements Closeable {
Review comment:
Could we add a comment for this class?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
BOOTSTRAP_SERVERS_CONFIG should be prefixed with REMOTE_LOG_METADATA_CLIENT_PREFIX, right?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
Review comment:
typo ard
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadataContext) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNo, remoteLogMetadataContext);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNo >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNo +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNo, null,
+ serde.serialize(remoteLogMetadataContext)), callback).get();
Review comment:
Sending one event at a time reduces the batching benefit in the producer. Could this handle the expected metadata load?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
Review comment:
Hmm, assignedMetaPartitions may not change if assignedTopicPartitions changes. Should we still update assignedTopicPartitions in that case?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ while (true) {
+ if (!initialized.get() && !close.get()) {
+ try {
+ //todo: There were race conditions observed in creating the remote log metadata topic when multiple
+ // brokers started running RLMM and creating the topic based on auto create flag.
+ // We may want to add a check here later.
+
+ // Create producer and consumer managers.
+ if (producerManager == null) {
Review comment:
Since producerManager and consumerManager are updated in a separate thread without holding lock, do they need to be volatile?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogMetadataTopicPartitioner.java
##########
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
+public class RemoteLogMetadataTopicPartitioner {
+ public static final Logger log = LoggerFactory.getLogger(RemoteLogMetadataTopicPartitioner.class);
+ private final int noOfMetadataTopicPartitions;
+
+ public RemoteLogMetadataTopicPartitioner(int noOfMetadataTopicPartitions) {
+ this.noOfMetadataTopicPartitions = noOfMetadataTopicPartitions;
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ Objects.requireNonNull(topicIdPartition, "TopicPartition can not be null");
+
+ int partitionNo = Utils.toPositive(Utils.murmur2(topicIdPartition.toString().getBytes(StandardCharsets.UTF_8)))
Review comment:
The toString() method could change over time. Perhaps it's safer to compute a customized hashcode for topicIdPartition here.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
+ initializationThread.start();
+ } finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ private void initializeResources() {
+ log.info("Initializing the resources.");
+ while (true) {
+ if (!initialized.get() && !close.get()) {
+ try {
+ //todo: There were race conditions observed in creating the remote log metadata topic when multiple
+ // brokers started running RLMM and creating the topic based on auto create flag.
+ // We may want to add a check here later.
+
+ // Create producer and consumer managers.
+ if (producerManager == null) {
+ producerManager = new ProducerManager(rlmmConfig, rlmmTopicPartitioner);
+ }
+
+ if (consumerManager == null) {
+ consumerManager = new ConsumerManager(rlmmConfig, remotePartitionMetadataStore, rlmmTopicPartitioner, time);
+ consumerManager.startConsumerThread();
+ }
+
+ initialized.set(true);
+ log.info("Initialized resources successfully.");
+ break;
+ } catch (Exception e) {
+ log.error("Encountered error while initializing producer/consumer", e);
+
+ // If it is already closing, break from here before it retries.
+ if (close.get()) {
+ break;
+ }
+
+ log.error("Sleep for : {} ms before it is retried again.", INITIALIZATION_RETRY_INTERVAL_MS);
+ Utils.sleep(INITIALIZATION_RETRY_INTERVAL_MS);
+ }
+ }
+ }
+ }
+
+ public boolean isInitialized() {
+ return initialized.get();
+ }
+
+ private void ensureInitializedAndNotClosed() {
+ if (close.get() || !initialized.get()) {
+ throw new IllegalStateException("This instance is in invalid state, initialized: " + initialized +
+ " close: " + close);
+ }
+ }
+
+ // Visible for testing.
+ void setTime(Time time) {
Review comment:
Should we pass in time through the constructor?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
Review comment:
Should we only process events corresponding to assignPartitions?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(TopicIdPartition topicIdPartition,
+ RemoteLogMetadata remoteLogMetadataContext) throws KafkaException {
+ ensureNotClosed();
+
+ int metadataPartitionNo = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNo, remoteLogMetadataContext);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNo >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNo +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNo, null,
+ serde.serialize(remoteLogMetadataContext)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() != null) {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
Review comment:
No need to cast to KafkaException.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
Review comment:
Do we need this since it's in the caller already?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
Review comment:
Hmm, shouldn't we start with assignedMetaPartitions?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ // Take these as configs with the respective default values.
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30_000L;
+
+ private volatile boolean configured = false;
+
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private ProducerManager producerManager;
+ private ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(topicIdPartition,
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
Review comment:
Waiting for each event to be consumed reduces throughput. Could we handle the expected metadata load with this?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
Review comment:
Do we need this? It seems that it's easier to just duplicate the property for producer and consumer.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
+ assignedTopicPartitions = Collections.unmodifiableSet(updatedReassignedPartitions);
+ assignedMetaPartitions = Collections.unmodifiableSet(updatedAssignedMetaPartitions);
Review comment:
If a partition is moved to a different broker, we will need to bootstrap the remote state for the partition by consuming from the beginning of the remote metadata partition. How is that handled?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
Review comment:
What's the purpose of waiting for consumption up to partitionToTargetEndOffsets?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
Review comment:
From which offset does the consumer start fetching?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663819484
##########
File path: storage/src/test/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerHarness.java
##########
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import kafka.api.IntegrationTestHarness;
+import kafka.utils.TestUtils;
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.utils.MockTime;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP;
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP;
+
+public class TopicBasedRemoteLogMetadataManagerHarness extends IntegrationTestHarness {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerHarness.class);
+
+ protected static final int METADATA_TOPIC_PARTITIONS_COUNT = 3;
+ protected static final int METADATA_TOPIC_REPLICATION_FACTOR = 2;
+ protected static final long METADATA_TOPIC_RETENTION_MS = 24 * 60 * 60 * 1000L;
+
+ private final Time time = new MockTime(1);
+ private TopicBasedRemoteLogMetadataManager topicBasedRemoteLogMetadataManager;
+
+ protected Map<String, Object> overrideRemoteLogMetadataManagerProps() {
+ return Collections.emptyMap();
+ }
+
+ public void initialize(Set<TopicIdPartition> topicIdPartitions) {
+ // Call setup to start the cluster.
+ super.setUp();
+
+ // Make sure the remote log metadata topic is created before it is used.
+ createMetadataTopic();
+
+ topicBasedRemoteLogMetadataManager = new TopicBasedRemoteLogMetadataManager(time);
+
+ // Initialize TopicBasedRemoteLogMetadataManager.
+ Map<String, Object> configs = new HashMap<>();
+ configs.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokerList());
+ configs.put("broker.id", 0);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, METADATA_TOPIC_PARTITIONS_COUNT);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, METADATA_TOPIC_REPLICATION_FACTOR);
+ configs.put(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, METADATA_TOPIC_RETENTION_MS);
+
+ log.debug("TopicBasedRemoteLogMetadataManager configs before adding overridden properties: {}", configs);
+ // Add override properties.
+ configs.putAll(overrideRemoteLogMetadataManagerProps());
+ log.debug("TopicBasedRemoteLogMetadataManager configs after adding overridden properties: {}", configs);
+
+ topicBasedRemoteLogMetadataManager.configure(configs);
+ try {
+ waitUntilInitialized(120_000);
Review comment:
I guess 60s may be sufficient, updated with that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r649390516
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ private volatile boolean close = false;
Review comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] kamalcph commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
kamalcph commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r665544995
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ *
+ * Caller of this class should take care of not sending messages once the closing of this instance is initiated.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not.
+ private final AtomicBoolean closing = new AtomicBoolean();
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
+ " must be less than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNum, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() == null) {
+ return callback.recordMetadata();
+ } else {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
+ } else {
+ throw new KafkaException(ex);
+ }
+ }
+ }
+
+ public void close() {
+ // If it is already closed, return from here.
+ if (!closing.compareAndSet(false, true)) {
Review comment:
This condition should be inverted.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657655419
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
Review comment:
`close/closing` is already volatile and its state is immediately reflected in other threads.
In `close()` method, the consumer is invoked with `wakeup()`, and the other thread may receive `WakeupException` if it is executing `poll()` or earlier than that. If it is after `poll` then the next check of `close/closing` allows to come out and finally close the consumer.
`updateAssignmentsForPartitions` and `close()` methods can not be called concurrently as it is alreadyhandled by `TopicBasedRemoteLogMetadataManager`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#issuecomment-868083433
Thanks @ccding for the review. Addressed them with the latest commit https://github.com/apache/kafka/commit/25c4ccb7c5ad44963a9708f0df67b982cf91ac15
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657657314
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
Review comment:
I did not add the checks for these methods as they will not be invoked in parallel when close is called. But I agree to have the guards here with read lock.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r648927106
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
Review comment:
As we discussed offline, we see the benefit of keeping several common client config like security to be shared across producer and consumer props avoiding any copy/paste mistakes.
User has an option not to use common client configs and use the respective producer and consumer configs.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] ccding commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
ccding commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r656735781
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
+
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private final Time time;
+ private final ConsumerTask consumerTask;
+ private final Thread consumerTaskThread;
+
+ public ConsumerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner,
+ Time time) {
+ this.rlmmConfig = rlmmConfig;
+ this.time = time;
+
+ //Create a task to consume messages and submit the respective events to RemotePartitionMetadataEventHandler.
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(rlmmConfig.consumerProperties());
+ consumerTask = new ConsumerTask(consumer, remotePartitionMetadataEventHandler, rlmmTopicPartitioner);
+ consumerTaskThread = KafkaThread.daemon("RLMMConsumerTask", consumerTask);
+ }
+
+ public void startConsumerThread() {
+ try {
+ // Start a thread to continuously consume records from topic partitions.
+ consumerTaskThread.start();
+ } catch (Exception e) {
+ throw new KafkaException("Error encountered while initializing and scheduling ConsumerTask thread", e);
+ }
+ }
+
+ /**
+ * Wait until the consumption reaches the offset of the metadata partition for the given {@code recordMetadata}.
+ *
+ * @param recordMetadata record metadata to be checked for consumption.
+ */
+ public void waitTillConsumptionCatchesUp(RecordMetadata recordMetadata) {
+ final int partition = recordMetadata.partition();
+
+ // If the current assignment does not have the subscription for this partition then return immediately.
+ if (!assignedPartition(partition)) {
+ log.warn("This consumer is not subscribed to the target partition [{}] on which message is produced.",
+ partition);
+ return;
+ }
+
+ final long offset = recordMetadata.offset();
+ long startTimeMillis = time.milliseconds();
+ while (true) {
+ long committedOffset = committedOffset(partition);
+ if (committedOffset >= offset) {
+ break;
+ }
+
+ log.debug("Committed offset [{}] for partition [{}], but the target offset: [{}], Sleeping for [{}] to retry again",
+ offset, partition, committedOffset, CONSUME_RECHECK_INTERVAL_MS);
+
+ if (time.milliseconds() - startTimeMillis > rlmmConfig.consumeWaitMs()) {
+ log.warn("Committed offset for partition:[{}] is : [{}], but the target offset: [{}] ",
+ partition, committedOffset, offset);
+ }
+
+ time.sleep(CONSUME_RECHECK_INTERVAL_MS);
+ }
+ }
+
+ private long committedOffset(int partition) {
+ return consumerTask.receivedOffsetForPartition(partition).orElse(-1L);
+ }
+
+ private boolean assignedPartition(int partition) {
+ return consumerTask.assignedPartition(partition);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // Consumer task will close the task and it internally closes all the resources including the consumer.
+ Utils.closeQuietly(consumerTask, "ConsumerTask");
+
+ // Wait until the consumer thread finishes.
+ try {
+ consumerTaskThread.join();
+ } catch (Exception e) {
+ log.error("Encountered error while waiting for consumerTaskThread to finish.", e);
+ }
+ }
+
+ public void addAssignmentsForPartitions(HashSet<TopicIdPartition> allPartitions) {
+ consumerTask.addAssignmentsForPartitions(allPartitions);
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
Review comment:
why is the variable name `partitions`, while the one above for `addAssignmentsForPartitions` is `allPartitions`?
also why one is `Set` and the other is `HashSet`?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
+
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private final Time time;
+ private final ConsumerTask consumerTask;
+ private final Thread consumerTaskThread;
+
+ public ConsumerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner,
+ Time time) {
+ this.rlmmConfig = rlmmConfig;
+ this.time = time;
+
+ //Create a task to consume messages and submit the respective events to RemotePartitionMetadataEventHandler.
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(rlmmConfig.consumerProperties());
+ consumerTask = new ConsumerTask(consumer, remotePartitionMetadataEventHandler, rlmmTopicPartitioner);
+ consumerTaskThread = KafkaThread.daemon("RLMMConsumerTask", consumerTask);
+ }
+
+ public void startConsumerThread() {
+ try {
+ // Start a thread to continuously consume records from topic partitions.
+ consumerTaskThread.start();
+ } catch (Exception e) {
+ throw new KafkaException("Error encountered while initializing and scheduling ConsumerTask thread", e);
+ }
+ }
+
+ /**
+ * Wait until the consumption reaches the offset of the metadata partition for the given {@code recordMetadata}.
+ *
+ * @param recordMetadata record metadata to be checked for consumption.
+ */
+ public void waitTillConsumptionCatchesUp(RecordMetadata recordMetadata) {
+ final int partition = recordMetadata.partition();
+
+ // If the current assignment does not have the subscription for this partition then return immediately.
+ if (!assignedPartition(partition)) {
+ log.warn("This consumer is not subscribed to the target partition [{}] on which message is produced.",
+ partition);
+ return;
+ }
+
+ final long offset = recordMetadata.offset();
+ long startTimeMillis = time.milliseconds();
+ while (true) {
+ long committedOffset = committedOffset(partition);
Review comment:
Can we avoid making the variable name the same as the function name?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
Review comment:
should this be fixed or configurable in `rlmmConfig`?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
Review comment:
why is this set to 30, rather than another number like 10, 50, 100?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ if (assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ } else {
+ log.debug("This event {} is skipped as the topic partition is not assigned for this instance.", remoteLogMetadata);
+ }
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
Review comment:
also here. Why are the variable names different? One is `updatedPartitions` and the other is `partitions`
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not. It will not accept any
+ // messages once it is set as true.
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNum, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() == null) {
+ return callback.recordMetadata();
+ } else {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
Review comment:
out of curiosity: why we don't do the check within the `new KafkaException(...)` call
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not. It will not accept any
+ // messages once it is set as true.
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNum, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() == null) {
+ return callback.recordMetadata();
+ } else {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
+ } else {
+ throw new KafkaException(ex);
+ }
+ }
+ }
+
+ private void ensureNotClosed() {
+ if (close) {
+ throw new IllegalStateException("This instance is already closed.");
+ }
+ }
+
+ public void close() {
+ // If it is already closed, return from here.
+ if (close) {
+ return;
+ }
+
+ close = true;
+
+ if (producer != null) {
Review comment:
is the check necessary?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
Review comment:
We may want a better variable name here. e.g., `isClosing` or `closed` or something else.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
Review comment:
`Id` -> `If`
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerManager.java
##########
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * This class manages the consumer thread viz {@link ConsumerTask} that polls messages from the assigned metadata topic partitions.
+ * It also provides a way to wait until the given record is received by the consumer before it is timed out with an interval of
+ * {@link TopicBasedRemoteLogMetadataManagerConfig#consumeWaitMs()}.
+ */
+public class ConsumerManager implements Closeable {
+
+ private static final Logger log = LoggerFactory.getLogger(ConsumerManager.class);
+ private static final long CONSUME_RECHECK_INTERVAL_MS = 50L;
+
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private final Time time;
+ private final ConsumerTask consumerTask;
+ private final Thread consumerTaskThread;
+
+ public ConsumerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner,
+ Time time) {
+ this.rlmmConfig = rlmmConfig;
+ this.time = time;
+
+ //Create a task to consume messages and submit the respective events to RemotePartitionMetadataEventHandler.
+ KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(rlmmConfig.consumerProperties());
+ consumerTask = new ConsumerTask(consumer, remotePartitionMetadataEventHandler, rlmmTopicPartitioner);
+ consumerTaskThread = KafkaThread.daemon("RLMMConsumerTask", consumerTask);
+ }
+
+ public void startConsumerThread() {
+ try {
+ // Start a thread to continuously consume records from topic partitions.
+ consumerTaskThread.start();
+ } catch (Exception e) {
+ throw new KafkaException("Error encountered while initializing and scheduling ConsumerTask thread", e);
+ }
+ }
+
+ /**
+ * Wait until the consumption reaches the offset of the metadata partition for the given {@code recordMetadata}.
+ *
+ * @param recordMetadata record metadata to be checked for consumption.
+ */
+ public void waitTillConsumptionCatchesUp(RecordMetadata recordMetadata) {
+ final int partition = recordMetadata.partition();
+
+ // If the current assignment does not have the subscription for this partition then return immediately.
+ if (!assignedPartition(partition)) {
+ log.warn("This consumer is not subscribed to the target partition [{}] on which message is produced.",
+ partition);
+ return;
+ }
+
+ final long offset = recordMetadata.offset();
+ long startTimeMillis = time.milliseconds();
Review comment:
I think in the codebase we use `Ms` more often than using `Millis`
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ if (assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ } else {
+ log.debug("This event {} is skipped as the topic partition is not assigned for this instance.", remoteLogMetadata);
+ }
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+ assignedTopicPartitions = Collections.unmodifiableSet(updatedReassignedPartitions);
+ log.debug("Assigned topic partitions: {}", assignedTopicPartitions);
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
+ assignedMetaPartitions = Collections.unmodifiableSet(updatedAssignedMetaPartitions);
+ log.debug("Assigned metadata topic partitions: {}", assignedMetaPartitions);
+ assignPartitions = true;
+ assignPartitionsLock.notifyAll();
+ } else {
+ log.debug("No change in assigned metadata topic partitions: {}", assignedMetaPartitions);
+ }
+ }
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int partition) {
+ return Optional.ofNullable(partitionToConsumedOffsets.get(partition));
+ }
+
+ public boolean assignedPartition(int partition) {
Review comment:
The function name is a little confusing with variable names in the same class, e.g., `assignPartitions`, `assignedTopicPartitions`. I am not sure if it would be better to rename `assignedPartition` to `isAssignedPartition`.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
Review comment:
multiple threads are reading and writing `close`, which is not thread safe
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
Review comment:
function names `addRemoteLogSegmentMetadata`, `updateRemoteLogSegmentMetadata`, and `putRemotePartitionDeleteMetadata` don't seem very consistent. Is there a way to improve it?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not. It will not accept any
+ // messages once it is set as true.
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
Review comment:
will the exception be caught by your own catch in line 74? maybe move it out of the `try` block?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogMetadataTopicPartitioner.java
##########
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+public class RemoteLogMetadataTopicPartitioner {
+ public static final Logger log = LoggerFactory.getLogger(RemoteLogMetadataTopicPartitioner.class);
+ private final int numMetadataTopicPartitions;
+
+ public RemoteLogMetadataTopicPartitioner(int numMetadataTopicPartitions) {
+ this.numMetadataTopicPartitions = numMetadataTopicPartitions;
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ Objects.requireNonNull(topicIdPartition, "TopicPartition can not be null");
+
+ int partitionNum = Utils.toPositive(Utils.murmur2(toBytes(topicIdPartition))) % numMetadataTopicPartitions;
+ log.debug("No of partitions [{}], partitionNum: [{}] for given topic: [{}]", numMetadataTopicPartitions, partitionNum, topicIdPartition);
+ return partitionNum;
+ }
+
+ private byte[] toBytes(TopicIdPartition topicIdPartition) {
+ // We do not want to depend upon hash code generation of Uuid as that may change.
+ int hash = Objects.hash(topicIdPartition.topicId().getLeastSignificantBits(),
+ topicIdPartition.topicId().getMostSignificantBits(),
+ topicIdPartition.topicPartition().partition());
+
+ return toBytes(hash);
+ }
+
+ private byte[] toBytes(int n) {
+ return new byte[]{
+ (byte) (n >> 24),
+ (byte) (n >> 16),
+ (byte) (n >> 8),
+ (byte) n
+ };
+ }
+}
Review comment:
new line at the end of the file
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
Review comment:
is the check necessary?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ if (assignedTopicPartitions.contains(remoteLogMetadata.topicIdPartition())) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ } else {
+ log.debug("This event {} is skipped as the topic partition is not assigned for this instance.", remoteLogMetadata);
+ }
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+ assignedTopicPartitions = Collections.unmodifiableSet(updatedReassignedPartitions);
+ log.debug("Assigned topic partitions: {}", assignedTopicPartitions);
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
+ assignedMetaPartitions = Collections.unmodifiableSet(updatedAssignedMetaPartitions);
+ log.debug("Assigned metadata topic partitions: {}", assignedMetaPartitions);
+ assignPartitions = true;
+ assignPartitionsLock.notifyAll();
+ } else {
+ log.debug("No change in assigned metadata topic partitions: {}", assignedMetaPartitions);
+ }
+ }
+ }
+
+ public Optional<Long> receivedOffsetForPartition(int partition) {
+ return Optional.ofNullable(partitionToConsumedOffsets.get(partition));
+ }
+
+ public boolean assignedPartition(int partition) {
+ return assignedMetaPartitions.contains(partition);
+ }
+
+ private void ensureNotClosed() {
+ if (close) {
+ throw new IllegalStateException("This instance is already closed");
+ }
+ }
+
+ public void close() {
+ if (!close) {
Review comment:
not good to use the same name for a variable and a function.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not. It will not accept any
+ // messages once it is set as true.
+ private volatile boolean close = false;
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ ensureNotClosed();
+
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
+ " is more than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
Review comment:
greater than or equal to?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ // It indicates whether the closing process has been started or not. Id it is set as true,
+ // consumer will stop consuming messages and it will not allow partition assignments to be updated.
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
Review comment:
can you explain what this variable means?
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+ ensureInitializedAndNotClosed();
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
Review comment:
Here has a race condition. It is possible `close=false` before calling `ensureInitializedAndNotClosed()` and the `close()` function call by another thread has completed before calling `remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch)`.
I think we should always grab a read lock before calling `ensureInitializedAndNotClosed();`, or do some other fancy things.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
+ assignedTopicPartitions = Collections.unmodifiableSet(updatedReassignedPartitions);
+ assignedMetaPartitions = Collections.unmodifiableSet(updatedAssignedMetaPartitions);
Review comment:
What is the cost of consuming from the beginning if the remote metadata partition grows huge?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r642095350
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
Review comment:
We have not yet added the code to store the consumed offset and start from those offsets whenever the consumer starts fetching from those partitions. We plan to add that in a subsequent PR.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
Review comment:
Right, I will add that check.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
+ log.info("Exiting from consumer task thread");
+ }
+
+ private void closeConsumer() {
+ log.info("Closing the consumer instance");
+ if (consumer != null) {
+ try {
+ consumer.close(Duration.ofSeconds(30));
+ } catch (Exception e) {
+ log.error("Error encountered while closing the consumer", e);
+ }
+ }
+ }
+
+ private Set<Integer> maybeWaitForPartitionsAssignment() {
+ Set<Integer> assignedMetaPartitionsSnapshot = Collections.emptySet();
+ synchronized (assignPartitionsLock) {
+ while (assignedMetaPartitions.isEmpty()) {
+ // If no partitions are assigned, wait until they are assigned.
+ log.info("Waiting for assigned remote log metadata partitions..");
+ try {
+ assignPartitionsLock.wait();
+ } catch (InterruptedException e) {
+ throw new KafkaException(e);
+ }
+ }
+
+ if (assignPartitions) {
+ assignedMetaPartitionsSnapshot = new HashSet<>(assignedMetaPartitions);
+ assignPartitions = false;
+ }
+ }
+ return assignedMetaPartitionsSnapshot;
+ }
+
+ private void handleRemoteLogMetadata(RemoteLogMetadata remoteLogMetadata) {
+ remotePartitionMetadataEventHandler.handleRemoteLogMetadata(remoteLogMetadata);
+ }
+
+ private void executeReassignment(Set<Integer> assignedMetaPartitionsSnapshot) {
+ Set<TopicPartition> assignedMetaTopicPartitions = assignedMetaPartitionsSnapshot.stream()
+ .map(partitionNum -> new TopicPartition(REMOTE_LOG_METADATA_TOPIC_NAME, partitionNum))
+ .collect(Collectors.toSet());
+ log.info("Reassigning partitions to consumer task [{}]", assignedMetaTopicPartitions);
+ consumer.assign(assignedMetaTopicPartitions);
+
+ log.debug("Fetching end offsets to consumer task [{}]", assignedMetaTopicPartitions);
+ Map<TopicPartition, Long> endOffsets;
+ while (true) {
+ try {
+ endOffsets = consumer.endOffsets(assignedMetaTopicPartitions, Duration.ofSeconds(30));
+ break;
+ } catch (Exception e) {
+ // ignore exception
+ log.debug("Error encountered in fetching end offsets", e);
+ }
+ }
+ log.debug("Fetched end offsets to consumer task [{}]", endOffsets);
+
+ for (Map.Entry<TopicPartition, Long> entry : endOffsets.entrySet()) {
+ if (entry.getValue() > 0) {
+ partitionToTargetEndOffsets.put(entry.getKey().partition(), entry.getValue());
+ }
+ }
+ }
+
+ public void addAssignmentsForPartitions(Set<TopicIdPartition> updatedPartitions) {
+ updateAssignmentsForPartitions(updatedPartitions, Collections.emptySet());
+ }
+
+ public void removeAssignmentsForPartitions(Set<TopicIdPartition> partitions) {
+ updateAssignmentsForPartitions(Collections.emptySet(), partitions);
+ }
+
+ private void updateAssignmentsForPartitions(Set<TopicIdPartition> addedPartitions,
+ Set<TopicIdPartition> removedPartitions) {
+ log.info("Updating assignments for addedPartitions: {} and removedPartition: {}", addedPartitions, removedPartitions);
+ ensureNotClosed();
+
+ Objects.requireNonNull(addedPartitions, "addedPartitions must not be null");
+ Objects.requireNonNull(removedPartitions, "removedPartitions must not be null");
+
+ if (addedPartitions.isEmpty() && removedPartitions.isEmpty()) {
+ return;
+ }
+
+ synchronized (assignPartitionsLock) {
+ Set<TopicIdPartition> updatedReassignedPartitions = new HashSet<>(assignedTopicPartitions);
+ updatedReassignedPartitions.addAll(addedPartitions);
+ updatedReassignedPartitions.removeAll(removedPartitions);
+ Set<Integer> updatedAssignedMetaPartitions = new HashSet<>();
+ for (TopicIdPartition tp : updatedReassignedPartitions) {
+ updatedAssignedMetaPartitions.add(topicPartitioner.metadataPartition(tp));
+ }
+
+ if (!updatedAssignedMetaPartitions.equals(assignedMetaPartitions)) {
+ assignedTopicPartitions = Collections.unmodifiableSet(updatedReassignedPartitions);
+ assignedMetaPartitions = Collections.unmodifiableSet(updatedAssignedMetaPartitions);
Review comment:
This is not addressed in this PR. I planned to have a followup PR for these changes. I may use a different consumer for the newly subscribed partitions to build the state.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
REMOTE_LOG_METADATA_CLIENT_PREFIX is just a prefix for generating client-ids for producer and consumer. If you are talking about the remote log metadata property prefix, It is assumed that the caller would have already removed those prefixes and sent the config. These prefixes are defined [here](https://github.com/apache/kafka/pull/10733/files/f185b4f86b457da35c21f1461fe1b1e1d73e4939#diff-31933b76c0578600db8744507e5c5ff840820d5a181a2a5d1e08c50bdc64ea6fR46)
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
+ if (bootstrapServers == null || bootstrapServers.isEmpty()) {
+ throw new IllegalArgumentException(BOOTSTRAP_SERVERS_CONFIG + " config must not be null or empty.");
+ }
+
+ consumeWaitMs = (long) parsedConfigs.get(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP);
+ metadataTopicPartitionsCount = (int) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP);
+ metadataTopicRetentionMillis = (long) parsedConfigs.get(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP);
+ if (metadataTopicRetentionMillis != -1 && metadataTopicRetentionMillis <= 0) {
+ throw new IllegalArgumentException("Invalid metadata topic retention in millis: " + metadataTopicRetentionMillis);
+ }
+
+ clientIdPrefix = REMOTE_LOG_METADATA_CLIENT_PREFIX + "_" + props.get(BROKER_ID);
+
+ initializeProducerConsumerProperties(props);
+ }
+
+ private void initializeProducerConsumerProperties(Map<String, ?> configs) {
+ Map<String, Object> commonClientConfigs = new HashMap<>();
+ commonClientConfigs.put(BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
+
+ Map<String, Object> producerOnlyConfigs = new HashMap<>();
+ Map<String, Object> consumerOnlyConfigs = new HashMap<>();
+
+ for (Map.Entry<String, ?> entry : configs.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX)) {
Review comment:
This is to avoid duplicate entries for both the producer and consumer. We added that in the KIP earlier.
If duplicating is the way we use at other places if any, I am fine with that.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that ard deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to target end offsets to be consumed.
+ private final Map<Integer, Long> partitionToTargetEndOffsets = new ConcurrentHashMap<>();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+
+ // Check whether messages are received till end offsets or not for the assigned metadata partitions.
+ if (!partitionToTargetEndOffsets.isEmpty()) {
+ for (Map.Entry<Integer, Long> entry : partitionToTargetEndOffsets.entrySet()) {
+ final Long offset = partitionToConsumedOffsets.getOrDefault(entry.getKey(), 0L);
+ if (offset >= entry.getValue()) {
+ partitionToTargetEndOffsets.remove(entry.getKey());
Review comment:
This is leftover code for other changes that I was working on for handling partition moving to a new broker in another branch. I will remove it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r665299398
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
##########
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.CommonClientConfigs;
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.kafka.clients.producer.ProducerConfig;
+import org.apache.kafka.common.config.ConfigDef;
+import org.apache.kafka.common.serialization.ByteArrayDeserializer;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG;
+import static org.apache.kafka.common.config.ConfigDef.Importance.LOW;
+import static org.apache.kafka.common.config.ConfigDef.Range.atLeast;
+import static org.apache.kafka.common.config.ConfigDef.Type.INT;
+import static org.apache.kafka.common.config.ConfigDef.Type.LONG;
+
+public final class TopicBasedRemoteLogMetadataManagerConfig {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManagerConfig.class.getName());
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_NAME = "__remote_log_metadata";
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP = "remote.log.metadata.topic.replication.factor";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP = "remote.log.metadata.topic.num.partitions";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP = "remote.log.metadata.topic.retention.ms";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP = "remote.log.metadata.publish.wait.ms";
+
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 60 * 1000L;
+
+ public static final String REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The number of partitions for remote log metadata Topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC = "Remote log metadata topic log retention in milli seconds." +
+ "Default: -1, that means unlimited. Users can configure this value based on their use cases. " +
+ "To avoid any data loss, this value should be more than the maximum retention period of any topic enabled with " +
+ "tiered storage in the cluster.";
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The amount of time in milli seconds to wait for the local consumer to " +
+ "receive the published event.";
+
+ public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX = "remote.log.metadata.common.client.";
+ public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX = "remote.log.metadata.producer.";
+ public static final String REMOTE_LOG_METADATA_CONSUMER_PREFIX = "remote.log.metadata.consumer.";
+
+ private static final String REMOTE_LOG_METADATA_CLIENT_PREFIX = "__remote_log_metadata_client";
+ private static final String BROKER_ID = "broker.id";
+
+ private static final ConfigDef CONFIG = new ConfigDef();
+ static {
+ CONFIG.define(REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT, DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
+ REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS_DOC)
+ .define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG, DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
+ REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC);
+ }
+
+ private final String clientIdPrefix;
+ private final int metadataTopicPartitionsCount;
+ private final String bootstrapServers;
+ private final long consumeWaitMs;
+ private final long metadataTopicRetentionMillis;
+
+ private Map<String, Object> consumerProps;
+ private Map<String, Object> producerProps;
+
+ public TopicBasedRemoteLogMetadataManagerConfig(Map<String, ?> props) {
+ log.info("Received props: [{}]", props);
+ Objects.requireNonNull(props, "props can not be null");
+
+ Map<String, Object> parsedConfigs = CONFIG.parse(props);
+
+ bootstrapServers = (String) props.get(BOOTSTRAP_SERVERS_CONFIG);
Review comment:
`RemoteLogMetadataManager.configure(Map<String, ?> configs)` is always invoked with stripping the RLMM prefix. "bootstrap.servers" property is sent as part of the configs here and any registered RLMM plugin will receive this property.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r649388017
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
Review comment:
I do not think that check is really needed here.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
+
+ // Remote log metadata topic partitions that consumer is assigned to.
+ private volatile Set<Integer> assignedMetaPartitions = Collections.emptySet();
+
+ // User topic partitions that this broker is a leader/follower for.
+ private Set<TopicIdPartition> assignedTopicPartitions = Collections.emptySet();
+
+ // Map of remote log metadata topic partition to consumed offsets.
+ private final Map<Integer, Long> partitionToConsumedOffsets = new ConcurrentHashMap<>();
+
+ public ConsumerTask(KafkaConsumer<byte[], byte[]> consumer,
+ RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler,
+ RemoteLogMetadataTopicPartitioner topicPartitioner) {
+ this.consumer = consumer;
+ this.remotePartitionMetadataEventHandler = remotePartitionMetadataEventHandler;
+ this.topicPartitioner = topicPartitioner;
+ }
+
+ @Override
+ public void run() {
+ log.info("Started Consumer task thread.");
+ try {
+ while (!close) {
+ Set<Integer> assignedMetaPartitionsSnapshot = maybeWaitForPartitionsAssignment();
+
+ if (!assignedMetaPartitionsSnapshot.isEmpty()) {
+ executeReassignment(assignedMetaPartitionsSnapshot);
+ }
+
+ log.info("Polling consumer to receive remote log metadata topic records");
+ ConsumerRecords<byte[], byte[]> consumerRecords
+ = consumer.poll(Duration.ofSeconds(POLL_INTERVAL_MS));
+ for (ConsumerRecord<byte[], byte[]> record : consumerRecords) {
+ handleRemoteLogMetadata(serde.deserialize(record.value()));
+ partitionToConsumedOffsets.put(record.partition(), record.offset());
+ }
+ }
+ } catch (Exception e) {
+ log.error("Error occurred in consumer task, close:[{}]", close, e);
+ }
+
+ closeConsumer();
Review comment:
It indicates whether the closing process has been started or not. If it is set as true, consumer will stop consuming messages and it will not allow partition assignments to be updated.
Updated the java doc of close.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import static org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_NAME;
+
+/**
+ * This class is responsible for consuming messages from remote log metadata topic ({@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME})
+ * partitions and maintain the state of the remote log segment metadata. It gives an API to add or remove
+ * for what topic partition's metadata should be consumed by this instance using
+ * {{@link #addAssignmentsForPartitions(Set)}} and {@link #removeAssignmentsForPartitions(Set)} respectively.
+ * <p>
+ * When a broker is started, controller sends topic partitions that this broker is leader or follower for and the
+ * partitions to be deleted. This class receives those notifications with
+ * {@link #addAssignmentsForPartitions(Set)} and {@link #removeAssignmentsForPartitions(Set)} assigns consumer for the
+ * respective remote log metadata partitions by using {@link RemoteLogMetadataTopicPartitioner#metadataPartition(TopicIdPartition)}.
+ * Any leadership changes later are called through the same API. We will remove the partitions that are deleted from
+ * this broker which are received through {@link #removeAssignmentsForPartitions(Set)}.
+ * <p>
+ * After receiving these events it invokes {@link RemotePartitionMetadataEventHandler#handleRemoteLogSegmentMetadata(RemoteLogSegmentMetadata)},
+ * which maintains in-memory representation of the state of {@link RemoteLogSegmentMetadata}.
+ */
+class ConsumerTask implements Runnable, Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ConsumerTask.class);
+
+ private static final long POLL_INTERVAL_MS = 30L;
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaConsumer<byte[], byte[]> consumer;
+ private final RemotePartitionMetadataEventHandler remotePartitionMetadataEventHandler;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+
+ private volatile boolean close = false;
+ private volatile boolean assignPartitions = false;
+
+ private final Object assignPartitionsLock = new Object();
Review comment:
I wanted to have a separate lock instance specifically for the assignments and the respective processing. It gives better clarity and separations even if we add any other logic by taking lock on this instance.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemoteLogMetadataTopicPartitioner.java
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.utils.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Objects;
+
+public class RemoteLogMetadataTopicPartitioner {
+ public static final Logger log = LoggerFactory.getLogger(RemoteLogMetadataTopicPartitioner.class);
+ private final int noOfMetadataTopicPartitions;
+
+ public RemoteLogMetadataTopicPartitioner(int noOfMetadataTopicPartitions) {
+ this.noOfMetadataTopicPartitions = noOfMetadataTopicPartitions;
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ Objects.requireNonNull(topicIdPartition, "TopicPartition can not be null");
+
+ int partitionNo = Utils.toPositive(Utils.murmur2(toBytes(topicIdPartition))) % noOfMetadataTopicPartitions;
+ log.debug("No of partitions [{}], partitionNo: [{}] for given topic: [{}]", noOfMetadataTopicPartitions, partitionNo, topicIdPartition);
+ return partitionNo;
+ }
+
+ private byte[] toBytes(TopicIdPartition topicIdPartition) {
+ // We do not want to depend upon hash code generation of Uuid as that may change.
+ int hash = Objects.hash(topicIdPartition.topicId().getLeastSignificantBits(),
+ topicIdPartition.topicId().getMostSignificantBits(),
+ topicIdPartition.topicPartition().topic(),
Review comment:
Good point. We can skip topic as topic-id is sufficient here.
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/RemotePartitionMetadataStore.java
##########
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentId;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteState;
+import org.apache.kafka.server.log.remote.storage.RemoteResourceNotFoundException;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class RemotePartitionMetadataStore extends RemotePartitionMetadataEventHandler implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(RemotePartitionMetadataStore.class);
+
+ private Map<TopicIdPartition, RemotePartitionDeleteMetadata> idToPartitionDeleteMetadata =
Review comment:
I went with assigning empty map as Map.clear() needs to go through all the entries and dereference them. Another way is to leave the map as it is and set the cose state and do not allow any operation when it is closed but it will have a check for each call.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664980913
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ConsumerTask.java
##########
@@ -220,16 +227,16 @@ public boolean isPartitionAssigned(int partition) {
return assignedMetaPartitions.contains(partition);
}
- private void ensureNotClosed() {
- if (closing) {
- throw new IllegalStateException("This instance is already closed");
- }
- }
-
public void close() {
if (!closing) {
- closing = true;
- consumer.wakeup();
+ synchronized (assignPartitionsLock) {
+ // Closing should be updated only after acquiring the lock to avoid race in
+ // maybeWaitForPartitionsAssignment() where it waits on assignPartitionsLock. It should not wait
+ // if the closing is already set.
+ closing = true;
Review comment:
`close()` will not be called concurrently here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r665555878
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/ProducerManager.java
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.Callback;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.server.log.remote.metadata.storage.serialization.RemoteLogMetadataSerde;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.time.Duration;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * This class is responsible for publishing messages into the remote log metadata topic partitions.
+ *
+ * Caller of this class should take care of not sending messages once the closing of this instance is initiated.
+ */
+public class ProducerManager implements Closeable {
+ private static final Logger log = LoggerFactory.getLogger(ProducerManager.class);
+
+ private final RemoteLogMetadataSerde serde = new RemoteLogMetadataSerde();
+ private final KafkaProducer<byte[], byte[]> producer;
+ private final RemoteLogMetadataTopicPartitioner topicPartitioner;
+ private final TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+
+ // It indicates whether closing process has been started or not.
+ private final AtomicBoolean closing = new AtomicBoolean();
+
+ public ProducerManager(TopicBasedRemoteLogMetadataManagerConfig rlmmConfig,
+ RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner) {
+ this.rlmmConfig = rlmmConfig;
+ this.producer = new KafkaProducer<>(rlmmConfig.producerProperties());
+ topicPartitioner = rlmmTopicPartitioner;
+ }
+
+ public RecordMetadata publishMessage(RemoteLogMetadata remoteLogMetadata) throws KafkaException {
+ TopicIdPartition topicIdPartition = remoteLogMetadata.topicIdPartition();
+ int metadataPartitionNum = topicPartitioner.metadataPartition(topicIdPartition);
+ log.debug("Publishing metadata message of partition:[{}] into metadata topic partition:[{}] with payload: [{}]",
+ topicIdPartition, metadataPartitionNum, remoteLogMetadata);
+ if (metadataPartitionNum >= rlmmConfig.metadataTopicPartitionsCount()) {
+ // This should never occur as long as metadata partitions always remain the same.
+ throw new KafkaException("Chosen partition no " + metadataPartitionNum +
+ " must be less than the partition count: " + rlmmConfig.metadataTopicPartitionsCount());
+ }
+
+ ProducerCallback callback = new ProducerCallback();
+ try {
+ producer.send(new ProducerRecord<>(rlmmConfig.remoteLogMetadataTopicName(), metadataPartitionNum, null,
+ serde.serialize(remoteLogMetadata)), callback).get();
+ } catch (KafkaException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new KafkaException("Exception occurred while publishing message for topicIdPartition: " + topicIdPartition, e);
+ }
+
+ if (callback.exception() == null) {
+ return callback.recordMetadata();
+ } else {
+ Exception ex = callback.exception();
+ if (ex instanceof KafkaException) {
+ throw (KafkaException) ex;
+ } else {
+ throw new KafkaException(ex);
+ }
+ }
+ }
+
+ public void close() {
+ // If it is already closed, return from here.
+ if (!closing.compareAndSet(false, true)) {
Review comment:
We do not really need this check here as it is already guarded in `TopicBasedRemoteLogMetadataManager`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r663809683
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,360 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(
+ remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
+ Set<TopicIdPartition> followerPartitions) {
+ Objects.requireNonNull(leaderPartitions, "leaderPartitions can not be null");
+ Objects.requireNonNull(followerPartitions, "followerPartitions can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ log.info("Received leadership notifications with leader partitions {} and follower partitions {}",
+ leaderPartitions, followerPartitions);
+
+ HashSet<TopicIdPartition> allPartitions = new HashSet<>(leaderPartitions);
+ allPartitions.addAll(followerPartitions);
+ consumerManager.addAssignmentsForPartitions(allPartitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void onStopPartitions(Set<TopicIdPartition> partitions) {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+ consumerManager.removeAssignmentsForPartitions(partitions);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void configure(Map<String, ?> configs) {
+ Objects.requireNonNull(configs, "configs can not be null.");
+
+ lock.writeLock().lock();
+ try {
+ if (configured) {
+ log.info("Skipping configure as it is already configured.");
+ return;
+ }
+
+ log.info("Started initializing with configs: {}", configs);
+
+ rlmmConfig = new TopicBasedRemoteLogMetadataManagerConfig(configs);
+ rlmmTopicPartitioner = new RemoteLogMetadataTopicPartitioner(rlmmConfig.metadataTopicPartitionsCount());
+ configured = true;
+ log.info("Successfully initialized with rlmmConfig: {}", rlmmConfig);
+
+ // Scheduling the initialization producer/consumer managers in a separate thread. Required resources may
+ // not yet be available now. This thread makes sure that it is retried at regular intervals until it is
+ // successful.
+ initializationThread = KafkaThread.daemon("RLMMInitializationThread", () -> initializeResources());
Review comment:
It was required to be retried until the topic is successfully created. I added the logic to check for topic creation too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r664014932
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,402 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.admin.AdminClient;
+import org.apache.kafka.clients.admin.NewTopic;
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.config.TopicConfig;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.errors.TopicExistsException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+ private static final long INITIALIZATION_RETRY_INTERVAL_MS = 30000L;
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean closing = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+ private final Time time = Time.SYSTEM;
+
+ private Thread initializationThread;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private volatile RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remotePartitionDeleteMetadata, "remotePartitionDeleteMetadata can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ doPublishMetadata(remotePartitionDeleteMetadata.topicIdPartition(), remotePartitionDeleteMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ private void doPublishMetadata(TopicIdPartition topicIdPartition, RemoteLogMetadata remoteLogMetadata)
+ throws RemoteStorageException {
+ log.debug("Publishing metadata for partition: [{}] with context: [{}]", topicIdPartition, remoteLogMetadata);
+
+ try {
+ // Publish the message to the topic.
+ RecordMetadata recordMetadata = producerManager.publishMessage(remoteLogMetadata);
+ // Wait until the consumer catches up with this offset. This will ensure read-after-write consistency
+ // semantics.
+ consumerManager.waitTillConsumptionCatchesUp(recordMetadata);
+ } catch (KafkaException e) {
+ if (e instanceof RetriableException) {
+ throw e;
+ } else {
+ throw new RemoteStorageException(e);
+ }
+ }
+ }
+
+ @Override
+ public Optional<RemoteLogSegmentMetadata> remoteLogSegmentMetadata(TopicIdPartition topicIdPartition,
+ int epochForOffset,
+ long offset)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.remoteLogSegmentMetadata(topicIdPartition, offset, epochForOffset);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Optional<Long> highestOffsetForEpoch(TopicIdPartition topicIdPartition,
+ int leaderEpoch)
+ throws RemoteStorageException {
+ lock.readLock().lock();
+ try {
+
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.highestLogOffset(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public Iterator<RemoteLogSegmentMetadata> listRemoteLogSegments(TopicIdPartition topicIdPartition, int leaderEpoch)
+ throws RemoteStorageException {
+ Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null");
+
+ lock.readLock().lock();
+ try {
+ ensureInitializedAndNotClosed();
+
+ return remotePartitionMetadataStore.listRemoteLogSegments(topicIdPartition, leaderEpoch);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ public int metadataPartition(TopicIdPartition topicIdPartition) {
+ return rlmmTopicPartitioner.metadataPartition(topicIdPartition);
+ }
+
+ // Visible For Testing
+ public Optional<Long> receivedOffsetForPartition(int metadataPartition) {
+ return consumerManager.receivedOffsetForPartition(metadataPartition);
+ }
+
+ @Override
+ public void onPartitionLeadershipChanges(Set<TopicIdPartition> leaderPartitions,
Review comment:
There can be a race here between this method and initializeResources() which I plan to address in a followup PR. I am working on a separate consumer to act on assigned partitions to cathcup. This race is being handled with those changes., same applies to onStopPartitions method too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: jira-unsubscribe@kafka.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [kafka] satishd commented on a change in pull request #10579: KAFKA-9555 Added default RLMM implementation based on internal topic storage.
Posted by GitBox <gi...@apache.org>.
satishd commented on a change in pull request #10579:
URL: https://github.com/apache/kafka/pull/10579#discussion_r657674359
##########
File path: storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManager.java
##########
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kafka.server.log.remote.metadata.storage;
+
+import org.apache.kafka.clients.producer.RecordMetadata;
+import org.apache.kafka.common.KafkaException;
+import org.apache.kafka.common.TopicIdPartition;
+import org.apache.kafka.common.errors.RetriableException;
+import org.apache.kafka.common.utils.KafkaThread;
+import org.apache.kafka.common.utils.Time;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogMetadataManager;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentMetadataUpdate;
+import org.apache.kafka.server.log.remote.storage.RemoteLogSegmentState;
+import org.apache.kafka.server.log.remote.storage.RemotePartitionDeleteMetadata;
+import org.apache.kafka.server.log.remote.storage.RemoteStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * This is the {@link RemoteLogMetadataManager} implementation with storage as an internal topic with name {@link TopicBasedRemoteLogMetadataManagerConfig#REMOTE_LOG_METADATA_TOPIC_NAME}.
+ * This is used to publish and fetch {@link RemoteLogMetadata} for the registered user topic partitions with
+ * {@link #onPartitionLeadershipChanges(Set, Set)}. Each broker will have an instance of this class and it subscribes
+ * to metadata updates for the registered user topic partitions.
+ */
+public class TopicBasedRemoteLogMetadataManager implements RemoteLogMetadataManager {
+ private static final Logger log = LoggerFactory.getLogger(TopicBasedRemoteLogMetadataManager.class);
+
+ private volatile boolean configured = false;
+
+ // It indicates whether the close process of this instance is started or not via #close() method.
+ // Using AtomicBoolean instead of volatile as it may encounter http://findbugs.sourceforge.net/bugDescriptions.html#SP_SPIN_ON_FIELD
+ // if the field is read but not updated in a spin loop like in #initializeResources() method.
+ private final AtomicBoolean close = new AtomicBoolean(false);
+ private final AtomicBoolean initialized = new AtomicBoolean(false);
+
+ private Thread initializationThread;
+ private Time time = Time.SYSTEM;
+ private volatile ProducerManager producerManager;
+ private volatile ConsumerManager consumerManager;
+
+ // This allows to gracefully close this instance using {@link #close()} method while there are some pending or new
+ // requests calling different methods which use the resources like producer/consumer managers.
+ private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+
+ private final RemotePartitionMetadataStore remotePartitionMetadataStore = new RemotePartitionMetadataStore();
+ private volatile TopicBasedRemoteLogMetadataManagerConfig rlmmConfig;
+ private RemoteLogMetadataTopicPartitioner rlmmTopicPartitioner;
+
+ public TopicBasedRemoteLogMetadataManager() {
+ }
+
+ // Visible for testing.
+ public TopicBasedRemoteLogMetadataManager(Time time) {
+ this.time = time;
+ }
+
+ @Override
+ public void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata)
+ throws RemoteStorageException {
+ Objects.requireNonNull(remoteLogSegmentMetadata, "remoteLogSegmentMetadata can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ // This allows gracefully rejecting the requests while closing of this instance is in progress, which triggers
+ // closing the producer/consumer manager instances.
+ lock.readLock().lock();
+ try {
+
+ // This method is allowed only to add remote log segment with the initial state(which is RemoteLogSegmentState.COPY_SEGMENT_STARTED)
+ // but not to update the existing remote log segment metadata.
+ if (remoteLogSegmentMetadata.state() != RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException(
+ "Given remoteLogSegmentMetadata should have state as " + RemoteLogSegmentState.COPY_SEGMENT_STARTED
+ + " but it contains state as: " + remoteLogSegmentMetadata.state());
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(remoteLogSegmentMetadata.remoteLogSegmentId().topicIdPartition(),
+ remoteLogSegmentMetadata);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate segmentMetadataUpdate)
+ throws RemoteStorageException {
+ Objects.requireNonNull(segmentMetadataUpdate, "segmentMetadataUpdate can not be null");
+
+ ensureInitializedAndNotClosed();
+
+ lock.readLock().lock();
+ try {
+ // Callers should use addRemoteLogSegmentMetadata to add RemoteLogSegmentMetadata with state as
+ // RemoteLogSegmentState.COPY_SEGMENT_STARTED.
+ if (segmentMetadataUpdate.state() == RemoteLogSegmentState.COPY_SEGMENT_STARTED) {
+ throw new IllegalArgumentException("Given remoteLogSegmentMetadata should not have the state as: "
+ + RemoteLogSegmentState.COPY_SEGMENT_STARTED);
+ }
+
+ // Publish the message to the topic.
+ doPublishMetadata(segmentMetadataUpdate.remoteLogSegmentId().topicIdPartition(), segmentMetadataUpdate);
+ } finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ @Override
+ public void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata)
Review comment:
Javadocs explain the behaviorr in detail.
`addRemoteLogSegmentMetadata` - adds a new entry.
`updateRemoteLogSegmentMetadata ` - updates an existing entry.
`putRemotePartitionDeleteMetadata ` - adds or updates an existing entry, put is generally used for that purpose. If this is not so clear, another option can be `addOrUpdateRemotePartitionDeleteMetadata`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org