You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ch...@apache.org on 2026/09/10 17:23:03 UTC

(kafka) branch 4.4 updated: KAFKA-20979: Ensure indexes resize are fsynced when shutting down (#23258)

This is an automated email from the ASF dual-hosted git repository.

chia7712 pushed a commit to branch 4.4
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/4.4 by this push:
     new beae700199c KAFKA-20979: Ensure indexes resize are fsynced when shutting down (#23258)
beae700199c is described below

commit beae700199c7ad805f8d5af53bdb7072cc1928e5
Author: Mickael Maison <mi...@users.noreply.github.com>
AuthorDate: Fri Aug 28 12:45:07 2026 +0200

    KAFKA-20979: Ensure indexes resize are fsynced when shutting down (#23258)
    
    This avoids reading invalid length for the index files after a clean
    shut down
    
    This does not prevent the issue happening on an unclean shutdown. For
    validating indexes at startup, we have
    https://issues.apache.org/jira/browse/KAFKA-19200
    
    Reviewers: Gaurav Narula <ga...@apple.com>, Luke Chen
     <sh...@gmail.com>
---
 .../kafka/storage/internals/log/AbstractIndex.java | 27 +++++++++++++++++-----
 .../kafka/storage/internals/log/LogSegment.java    |  8 +++----
 .../storage/internals/log/AbstractIndexTest.java   | 17 ++++++++++++++
 3 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/storage/src/main/java/org/apache/kafka/storage/internals/log/AbstractIndex.java b/storage/src/main/java/org/apache/kafka/storage/internals/log/AbstractIndex.java
index e484d88bcac..4af93d52a40 100644
--- a/storage/src/main/java/org/apache/kafka/storage/internals/log/AbstractIndex.java
+++ b/storage/src/main/java/org/apache/kafka/storage/internals/log/AbstractIndex.java
@@ -198,6 +198,15 @@ public abstract class AbstractIndex implements Closeable {
      * @return a boolean indicating whether the size of the memory map and the underneath file is changed or not.
      */
     public boolean resize(int newSize) throws IOException {
+        return resize(newSize, false);
+    }
+
+    /**
+     * @param newSize new size of the index file
+     * @param sync if true, fsync the file after resizing to ensure the size is durable
+     * @return true if the index was resized, false otherwise
+     */
+    public boolean resize(int newSize, boolean sync) throws IOException {
         return inLock(() ->
                 inRemapWriteLock(() -> {
                     int roundedNewSize = roundDownToExactMultiple(newSize, entrySize());
@@ -216,6 +225,9 @@ public abstract class AbstractIndex implements Closeable {
                             mmap = raf.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, roundedNewSize);
                             this.maxEntries = mmap.limit() / entrySize();
                             mmap.position(position);
+                            if (sync) {
+                                raf.getChannel().force(true);
+                            }
                             log.debug("Resized {} to {}, position is {} and limit is {}", file.getAbsolutePath(), roundedNewSize,
                                     mmap.position(), mmap.limit());
                             return true;
@@ -244,7 +256,9 @@ public abstract class AbstractIndex implements Closeable {
      */
     public void flush() {
         inLock(() -> {
-            mmap.force();
+            if (mmap != null) {
+                mmap.force();
+            }
         });
     }
 
@@ -261,13 +275,13 @@ public abstract class AbstractIndex implements Closeable {
     }
 
     /**
-     * Trim this segment to fit just the valid entries, deleting all trailing unwritten bytes from
-     * the file.
+     * Trim this index to fit just the valid entries, deleting all trailing unwritten bytes from the file.
+     * @param sync if true, fsync the file after resizing to ensure both content and size are durable
      */
-    public void trimToValidSize() throws IOException {
+    public void trimToValidSize(boolean sync) throws IOException {
         inLock(() -> {
             if (mmap != null) {
-                resize(entrySize() * entries);
+                resize(entrySize() * entries, sync);
             }
         });
     }
@@ -280,7 +294,8 @@ public abstract class AbstractIndex implements Closeable {
     }
 
     public void close() throws IOException {
-        trimToValidSize();
+        flush(); // Ensure the index content is flushed to disk as LogSegment.close may append an entry
+        trimToValidSize(true);
         closeHandler();
     }
 
diff --git a/storage/src/main/java/org/apache/kafka/storage/internals/log/LogSegment.java b/storage/src/main/java/org/apache/kafka/storage/internals/log/LogSegment.java
index 25b7a3fbb54..0effe0ecd4d 100644
--- a/storage/src/main/java/org/apache/kafka/storage/internals/log/LogSegment.java
+++ b/storage/src/main/java/org/apache/kafka/storage/internals/log/LogSegment.java
@@ -521,10 +521,10 @@ public class LogSegment implements Closeable {
             LOGGER.debug("Truncated {} invalid bytes at the end of segment {} during recovery", truncated, log.file().getAbsolutePath());
 
         log.truncateTo(validBytes);
-        offsetIndex().trimToValidSize();
+        offsetIndex().trimToValidSize(false);
         // A normally closed segment always appends the biggest timestamp ever seen into log segment, we do this as well.
         timeIndex().maybeAppend(maxTimestampSoFar(), shallowOffsetOfMaxTimestampSoFar(), true);
-        timeIndex().trimToValidSize();
+        timeIndex().trimToValidSize(false);
         return truncated;
     }
 
@@ -685,8 +685,8 @@ public class LogSegment implements Closeable {
      */
     public void onBecomeInactiveSegment() throws IOException {
         timeIndex().maybeAppend(maxTimestampSoFar(), shallowOffsetOfMaxTimestampSoFar(), true);
-        offsetIndex().trimToValidSize();
-        timeIndex().trimToValidSize();
+        offsetIndex().trimToValidSize(false);
+        timeIndex().trimToValidSize(false);
         log.trim();
     }
 
diff --git a/storage/src/test/java/org/apache/kafka/storage/internals/log/AbstractIndexTest.java b/storage/src/test/java/org/apache/kafka/storage/internals/log/AbstractIndexTest.java
index f6b28345b93..1378577fa95 100644
--- a/storage/src/test/java/org/apache/kafka/storage/internals/log/AbstractIndexTest.java
+++ b/storage/src/test/java/org/apache/kafka/storage/internals/log/AbstractIndexTest.java
@@ -35,11 +35,18 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 public class AbstractIndexTest {
     private static class TestIndex extends AbstractIndex {
         private boolean unmapInvoked = false;
+        private boolean flushInvoked = false;
         private MappedByteBuffer unmappedBuffer = null;
         public TestIndex(File file, long baseOffset, int maxIndexSize, boolean writable) throws IOException {
             super(file, baseOffset, maxIndexSize, writable);
         }
 
+        @Override
+        public void flush() {
+            flushInvoked = true;
+            super.flush();
+        }
+
         @Override
         protected int entrySize() {
             return 1;
@@ -86,4 +93,14 @@ public class AbstractIndexTest {
         assertSame(oldMmap, idx.unmappedBuffer, "old mmap should be unmapped");
         assertNotSame(idx.unmappedBuffer, idx.mmap());
     }
+
+    @Test
+    public void testCloseInvokeFlush() throws IOException {
+        File f = new File(TestUtils.tempDirectory(), "test-index");
+        TestIndex idx = new TestIndex(f, 0L, 100, true);
+        assertFalse(idx.flushInvoked, "Flush should not have been invoked yet");
+
+        idx.close();
+        assertTrue(idx.flushInvoked, "Flush should have been invoked during close");
+    }
 }