You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ch...@apache.org on 2026/09/10 17:23:03 UTC
(kafka) branch 4.4 updated: KAFKA-20979: Ensure indexes resize are fsynced when shutting down (#23258)
This is an automated email from the ASF dual-hosted git repository.
chia7712 pushed a commit to branch 4.4
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/4.4 by this push:
new beae700199c KAFKA-20979: Ensure indexes resize are fsynced when shutting down (#23258)
beae700199c is described below
commit beae700199c7ad805f8d5af53bdb7072cc1928e5
Author: Mickael Maison <mi...@users.noreply.github.com>
AuthorDate: Fri Aug 28 12:45:07 2026 +0200
KAFKA-20979: Ensure indexes resize are fsynced when shutting down (#23258)
This avoids reading invalid length for the index files after a clean
shut down
This does not prevent the issue happening on an unclean shutdown. For
validating indexes at startup, we have
https://issues.apache.org/jira/browse/KAFKA-19200
Reviewers: Gaurav Narula <ga...@apple.com>, Luke Chen
<sh...@gmail.com>
---
.../kafka/storage/internals/log/AbstractIndex.java | 27 +++++++++++++++++-----
.../kafka/storage/internals/log/LogSegment.java | 8 +++----
.../storage/internals/log/AbstractIndexTest.java | 17 ++++++++++++++
3 files changed, 42 insertions(+), 10 deletions(-)
diff --git a/storage/src/main/java/org/apache/kafka/storage/internals/log/AbstractIndex.java b/storage/src/main/java/org/apache/kafka/storage/internals/log/AbstractIndex.java
index e484d88bcac..4af93d52a40 100644
--- a/storage/src/main/java/org/apache/kafka/storage/internals/log/AbstractIndex.java
+++ b/storage/src/main/java/org/apache/kafka/storage/internals/log/AbstractIndex.java
@@ -198,6 +198,15 @@ public abstract class AbstractIndex implements Closeable {
* @return a boolean indicating whether the size of the memory map and the underneath file is changed or not.
*/
public boolean resize(int newSize) throws IOException {
+ return resize(newSize, false);
+ }
+
+ /**
+ * @param newSize new size of the index file
+ * @param sync if true, fsync the file after resizing to ensure the size is durable
+ * @return true if the index was resized, false otherwise
+ */
+ public boolean resize(int newSize, boolean sync) throws IOException {
return inLock(() ->
inRemapWriteLock(() -> {
int roundedNewSize = roundDownToExactMultiple(newSize, entrySize());
@@ -216,6 +225,9 @@ public abstract class AbstractIndex implements Closeable {
mmap = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, roundedNewSize);
this.maxEntries = mmap.limit() / entrySize();
mmap.position(position);
+ if (sync) {
+ raf.getChannel().force(true);
+ }
log.debug("Resized {} to {}, position is {} and limit is {}", file.getAbsolutePath(), roundedNewSize,
mmap.position(), mmap.limit());
return true;
@@ -244,7 +256,9 @@ public abstract class AbstractIndex implements Closeable {
*/
public void flush() {
inLock(() -> {
- mmap.force();
+ if (mmap != null) {
+ mmap.force();
+ }
});
}
@@ -261,13 +275,13 @@ public abstract class AbstractIndex implements Closeable {
}
/**
- * Trim this segment to fit just the valid entries, deleting all trailing unwritten bytes from
- * the file.
+ * Trim this index to fit just the valid entries, deleting all trailing unwritten bytes from the file.
+ * @param sync if true, fsync the file after resizing to ensure both content and size are durable
*/
- public void trimToValidSize() throws IOException {
+ public void trimToValidSize(boolean sync) throws IOException {
inLock(() -> {
if (mmap != null) {
- resize(entrySize() * entries);
+ resize(entrySize() * entries, sync);
}
});
}
@@ -280,7 +294,8 @@ public abstract class AbstractIndex implements Closeable {
}
public void close() throws IOException {
- trimToValidSize();
+ flush(); // Ensure the index content is flushed to disk as LogSegment.close may append an entry
+ trimToValidSize(true);
closeHandler();
}
diff --git a/storage/src/main/java/org/apache/kafka/storage/internals/log/LogSegment.java b/storage/src/main/java/org/apache/kafka/storage/internals/log/LogSegment.java
index 25b7a3fbb54..0effe0ecd4d 100644
--- a/storage/src/main/java/org/apache/kafka/storage/internals/log/LogSegment.java
+++ b/storage/src/main/java/org/apache/kafka/storage/internals/log/LogSegment.java
@@ -521,10 +521,10 @@ public class LogSegment implements Closeable {
LOGGER.debug("Truncated {} invalid bytes at the end of segment {} during recovery", truncated, log.file().getAbsolutePath());
log.truncateTo(validBytes);
- offsetIndex().trimToValidSize();
+ offsetIndex().trimToValidSize(false);
// A normally closed segment always appends the biggest timestamp ever seen into log segment, we do this as well.
timeIndex().maybeAppend(maxTimestampSoFar(), shallowOffsetOfMaxTimestampSoFar(), true);
- timeIndex().trimToValidSize();
+ timeIndex().trimToValidSize(false);
return truncated;
}
@@ -685,8 +685,8 @@ public class LogSegment implements Closeable {
*/
public void onBecomeInactiveSegment() throws IOException {
timeIndex().maybeAppend(maxTimestampSoFar(), shallowOffsetOfMaxTimestampSoFar(), true);
- offsetIndex().trimToValidSize();
- timeIndex().trimToValidSize();
+ offsetIndex().trimToValidSize(false);
+ timeIndex().trimToValidSize(false);
log.trim();
}
diff --git a/storage/src/test/java/org/apache/kafka/storage/internals/log/AbstractIndexTest.java b/storage/src/test/java/org/apache/kafka/storage/internals/log/AbstractIndexTest.java
index f6b28345b93..1378577fa95 100644
--- a/storage/src/test/java/org/apache/kafka/storage/internals/log/AbstractIndexTest.java
+++ b/storage/src/test/java/org/apache/kafka/storage/internals/log/AbstractIndexTest.java
@@ -35,11 +35,18 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class AbstractIndexTest {
private static class TestIndex extends AbstractIndex {
private boolean unmapInvoked = false;
+ private boolean flushInvoked = false;
private MappedByteBuffer unmappedBuffer = null;
public TestIndex(File file, long baseOffset, int maxIndexSize, boolean writable) throws IOException {
super(file, baseOffset, maxIndexSize, writable);
}
+ @Override
+ public void flush() {
+ flushInvoked = true;
+ super.flush();
+ }
+
@Override
protected int entrySize() {
return 1;
@@ -86,4 +93,14 @@ public class AbstractIndexTest {
assertSame(oldMmap, idx.unmappedBuffer, "old mmap should be unmapped");
assertNotSame(idx.unmappedBuffer, idx.mmap());
}
+
+ @Test
+ public void testCloseInvokeFlush() throws IOException {
+ File f = new File(TestUtils.tempDirectory(), "test-index");
+ TestIndex idx = new TestIndex(f, 0L, 100, true);
+ assertFalse(idx.flushInvoked, "Flush should not have been invoked yet");
+
+ idx.close();
+ assertTrue(idx.flushInvoked, "Flush should have been invoked during close");
+ }
}