You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by cm...@apache.org on 2024/02/14 00:50:09 UTC
(kafka) 01/01: MINOR: update leaderAndEpoch before initializing metadata publishers
This is an automated email from the ASF dual-hosted git repository.
cmccabe pushed a commit to branch init_metadata_publishers
in repository https://gitbox.apache.org/repos/asf/kafka.git
commit b31840f8d7bd6a88da4a7a1f190b0ab438b817ef
Author: Colin P. McCabe <cm...@apache.org>
AuthorDate: Tue Feb 13 16:47:43 2024 -0800
MINOR: update leaderAndEpoch before initializing metadata publishers
---
.../src/main/scala/kafka/server/SharedServer.scala | 1 +
.../apache/kafka/image/loader/MetadataLoader.java | 23 ++++++++++++++++++----
.../kafka/image/loader/MetadataLoaderTest.java | 10 ++++++++++
3 files changed, 30 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/kafka/server/SharedServer.scala b/core/src/main/scala/kafka/server/SharedServer.scala
index 31b4957e3f5..4dac53ed292 100644
--- a/core/src/main/scala/kafka/server/SharedServer.scala
+++ b/core/src/main/scala/kafka/server/SharedServer.scala
@@ -285,6 +285,7 @@ class SharedServer(
setThreadNamePrefix(s"kafka-${sharedServerConfig.nodeId}-").
setFaultHandler(metadataLoaderFaultHandler).
setHighWaterMarkAccessor(() => _raftManager.client.highWatermark()).
+ setLeaderAndEpochAccessor(() => _raftManager.client.leaderAndEpoch()).
setMetrics(metadataLoaderMetrics)
loader = loaderBuilder.build()
snapshotEmitter = new SnapshotEmitter.Builder().
diff --git a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java
index c6194f4e6dd..e967228a59e 100644
--- a/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java
+++ b/metadata/src/main/java/org/apache/kafka/image/loader/MetadataLoader.java
@@ -75,6 +75,7 @@ public class MetadataLoader implements RaftClient.Listener<ApiMessageAndVersion>
private FaultHandler faultHandler = (m, e) -> new FaultHandlerException(m, e);
private MetadataLoaderMetrics metrics = null;
private Supplier<OptionalLong> highWaterMarkAccessor = null;
+ private Supplier<LeaderAndEpoch> leaderAndEpochAccessor = () -> LeaderAndEpoch.UNKNOWN;
public Builder setNodeId(int nodeId) {
this.nodeId = nodeId;
@@ -101,6 +102,11 @@ public class MetadataLoader implements RaftClient.Listener<ApiMessageAndVersion>
return this;
}
+ public Builder setLeaderAndEpochAccessor(Supplier<LeaderAndEpoch> leaderAndEpochAccessor) {
+ this.leaderAndEpochAccessor = leaderAndEpochAccessor;
+ return this;
+ }
+
public Builder setMetrics(MetadataLoaderMetrics metrics) {
this.metrics = metrics;
return this;
@@ -126,7 +132,8 @@ public class MetadataLoader implements RaftClient.Listener<ApiMessageAndVersion>
threadNamePrefix,
faultHandler,
metrics,
- highWaterMarkAccessor);
+ highWaterMarkAccessor,
+ leaderAndEpochAccessor);
}
}
@@ -157,6 +164,11 @@ public class MetadataLoader implements RaftClient.Listener<ApiMessageAndVersion>
*/
private final Supplier<OptionalLong> highWaterMarkAccessor;
+ /**
+ * A function which supplies the current leader and epoch.
+ */
+ private final Supplier<LeaderAndEpoch> leaderAndEpochAccessor;
+
/**
* Publishers which haven't received any metadata yet.
*/
@@ -197,13 +209,15 @@ public class MetadataLoader implements RaftClient.Listener<ApiMessageAndVersion>
String threadNamePrefix,
FaultHandler faultHandler,
MetadataLoaderMetrics metrics,
- Supplier<OptionalLong> highWaterMarkAccessor
+ Supplier<OptionalLong> highWaterMarkAccessor,
+ Supplier<LeaderAndEpoch> leaderAndEpochAccessor
) {
this.log = logContext.logger(MetadataLoader.class);
this.time = time;
this.faultHandler = faultHandler;
this.metrics = metrics;
this.highWaterMarkAccessor = highWaterMarkAccessor;
+ this.leaderAndEpochAccessor = leaderAndEpochAccessor;
this.uninitializedPublishers = new LinkedHashMap<>();
this.publishers = new LinkedHashMap<>();
this.image = MetadataImage.EMPTY;
@@ -245,8 +259,9 @@ public class MetadataLoader implements RaftClient.Listener<ApiMessageAndVersion>
offset + " and high water mark is {}", where, highWaterMark.getAsLong());
return true;
}
- log.info("{}: The loader finished catching up to the current high water mark of {}",
- where, highWaterMark.getAsLong());
+ currentLeaderAndEpoch = leaderAndEpochAccessor.get();
+ log.info("{}: The loader finished catching up to the current high water mark of {}, {}",
+ where, highWaterMark.getAsLong(), currentLeaderAndEpoch);
catchingUp = false;
return false;
}
diff --git a/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java
index 921c241a09a..ad771c7bfb3 100644
--- a/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java
+++ b/metadata/src/test/java/org/apache/kafka/image/loader/MetadataLoaderTest.java
@@ -55,6 +55,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.OptionalInt;
import java.util.OptionalLong;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
@@ -97,6 +98,7 @@ public class MetadataLoaderTest {
volatile LogDeltaManifest latestLogDeltaManifest = null;
volatile SnapshotManifest latestSnapshotManifest = null;
volatile boolean closed = false;
+ volatile LeaderAndEpoch currentLeaderAndEpoch = LeaderAndEpoch.UNKNOWN;
MockPublisher() {
this("MockPublisher");
@@ -132,6 +134,11 @@ public class MetadataLoaderTest {
firstPublish.complete(null);
}
+ @Override
+ public void onControllerChange(LeaderAndEpoch newLeaderAndEpoch) {
+ currentLeaderAndEpoch = newLeaderAndEpoch;
+ }
+
@Override
public void close() throws Exception {
firstPublish.completeExceptionally(new RejectedExecutionException());
@@ -549,6 +556,7 @@ public class MetadataLoaderTest {
try (MetadataLoader loader = new MetadataLoader.Builder().
setFaultHandler(faultHandler).
setHighWaterMarkAccessor(() -> highWaterMark.get()).
+ setLeaderAndEpochAccessor(() -> new LeaderAndEpoch(OptionalInt.of(123), 456)).
build()) {
loader.installPublishers(publishers).get();
loadTestSnapshot(loader, 200);
@@ -568,6 +576,8 @@ public class MetadataLoaderTest {
loadTestSnapshot(loader, 220);
assertEquals(220L, loader.lastAppliedOffset());
publishers.get(0).firstPublish.get(1, TimeUnit.MINUTES);
+ assertEquals(new LeaderAndEpoch(OptionalInt.of(123), 456),
+ publishers.get(0).currentLeaderAndEpoch);
}
faultHandler.maybeRethrowFirstException();
}