You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by js...@apache.org on 2026/10/14 07:04:28 UTC

(jackrabbit-oak) branch trunk updated: OAK-12404 - CompactionWriter.flush() persists partial writer before target writer (#3134)

This is an automated email from the ASF dual-hosted git repository.

jsedding pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 1ebb6c2934 OAK-12404 - CompactionWriter.flush() persists partial writer before target writer (#3134)
1ebb6c2934 is described below

commit 1ebb6c2934d3f179d1384f64f4e439691f6a5534
Author: Julian Sedding <js...@apache.org>
AuthorDate: Wed Oct 14 09:04:21 2026 +0200

    OAK-12404 - CompactionWriter.flush() persists partial writer before target writer (#3134)
    
    - flush the target writer before the partial writer
---
 .../oak/segment/file/CompactionWriter.java         |   2 +-
 .../file/CompactionWriterFlushOrderTest.java       | 172 +++++++++++++++++++++
 2 files changed, 173 insertions(+), 1 deletion(-)

diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionWriter.java
index c8fb1f6e5e..200205e125 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionWriter.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionWriter.java
@@ -87,8 +87,8 @@ public class CompactionWriter {
     }
 
     public void flush() throws IOException {
-        partialWriter.flush();
         targetWriter.flush();
+        partialWriter.flush();
     }
 
     public @Nullable FullyCompactedNodeState getPreviouslyCompactedState(NodeState nodeState) {
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/CompactionWriterFlushOrderTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/CompactionWriterFlushOrderTest.java
new file mode 100644
index 0000000000..51a5e462fd
--- /dev/null
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/CompactionWriterFlushOrderTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.file;
+
+import static org.apache.jackrabbit.oak.segment.CompactorTestUtils.addTestContent;
+import static org.apache.jackrabbit.oak.segment.DefaultSegmentWriterBuilder.defaultSegmentWriterBuilder;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import org.apache.jackrabbit.oak.segment.ClassicCompactor;
+import org.apache.jackrabbit.oak.segment.Compactor;
+import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.SegmentId;
+import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.SegmentWriterFactory;
+import org.apache.jackrabbit.oak.segment.file.cancel.Canceller;
+import org.apache.jackrabbit.oak.segment.memory.MemoryStore;
+import org.apache.jackrabbit.oak.segment.spi.persistence.GCGeneration;
+import org.apache.jackrabbit.oak.spi.gc.GCMonitor;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * {@link CompactionWriter#flush()} flushes the partial writer before the target writer:
+ * <pre>
+ * public void flush() throws IOException {
+ *     partialWriter.flush();
+ *     targetWriter.flush();
+ * }
+ * </pre>
+ * A partially compacted node references the fully compacted nodes below it, so this
+ * persists a segment before the segment it depends on. Two consequences:
+ * <ul>
+ * <li>A crash between the two flushes leaves a persisted node referencing records that
+ *     were never written.</li>
+ * <li>The segment holding the partial root is the compacted root of a
+ *     {@link CompactionResult#partiallySucceeded partially succeeded} compaction, and
+ *     {@link DefaultCleanupContext#shouldReclaim} reclaims every compacted segment
+ *     persisted <em>after</em> the compacted root ("dangling future segments"). The
+ *     target segment flushed afterwards matches that description while still being
+ *     referenced from the head.</li>
+ * </ul>
+ * This only matters for soft cancellation: without it the partial writer has nothing
+ * buffered and the flush order is unobservable.
+ */
+public class CompactionWriterFlushOrderTest {
+
+    private final List<SegmentId> persisted = new ArrayList<>();
+
+    private RecordingStore store;
+
+    @BeforeEach
+    public void setUp() throws IOException {
+        store = new RecordingStore();
+    }
+
+    /**
+     * Drives a {@link ClassicCompactor} with a soft canceller, which routes the completed
+     * subtrees to the target writer and the spine above the cancellation point to the
+     * partial writer. Asserts that the segment holding the compacted root does not
+     * reference any segment that is persisted after it.
+     */
+    @Test
+    public void softCancellationPersistsTheSpineBeforeTheContentItReferences() throws Exception {
+        NodeStore nodeStore = SegmentNodeStoreBuilders.builder(store).build();
+        for (int i = 0; i < 100; i++) {
+            addTestContent("node" + i, nodeStore, 42);
+        }
+
+        SegmentNodeState head = store.getReader().readHeadState(store.getRevisions());
+        GCGeneration base = head.getGcGeneration();
+        GCIncrement increment = new GCIncrement(base, base.nextPartial(), base.nextFull());
+
+        SegmentWriterFactory writerFactory = generation -> defaultSegmentWriterBuilder("c")
+                .withGeneration(generation)
+                .build(store);
+        CompactionWriter writer = new CompactionWriter(store.getReader(), null, increment, writerFactory);
+
+        GCNodeWriteMonitor monitor = new GCNodeWriteMonitor(-1, GCMonitor.EMPTY);
+        Compactor compactor = new ClassicCompactor(writer, monitor);
+
+        Canceller softCanceller = Canceller.newCanceller()
+                .withCondition("40 nodes compacted", () -> monitor.getCompactedNodes() >= 40);
+        CompactedNodeState compacted =
+                compactor.compactDown(head, Canceller.newCanceller(), softCanceller);
+
+        assertNotNull(compacted, "compaction should not have been hard cancelled");
+        assertFalse(compacted.isComplete(), "compaction should have been soft cancelled");
+
+        SegmentId rootSegment = compacted.getRecordId().getSegmentId();
+        assertEquals(base.nextPartial(), rootSegment.getGcGeneration(),
+                "the partial root should be at the partial generation");
+
+        writer.flush();
+
+        int rootIndex = persisted.indexOf(rootSegment);
+        assertTrue(rootIndex >= 0, "the partial root segment should have been persisted");
+
+        // The segment reference table is what TarReader#mark builds the cleanup graph from.
+        Set<UUID> referencedByRoot = referencedSegmentIds(store.readSegment(rootSegment));
+
+        // Segments the spine depends on, yet which reach the store only after it. They are
+        // at the target generation, i.e. compacted, which is also what
+        // DefaultCleanupContext#isDanglingFutureSegment reclaims without consulting
+        // reachability: it assumes compacted segments are unused iff they are persisted
+        // after the last compacted root.
+        List<SegmentId> referencedButLater = persisted.subList(rootIndex + 1, persisted.size())
+                .stream()
+                .filter(id -> referencedByRoot.contains(id.asUUID()))
+                .collect(Collectors.toList());
+
+        assertTrue(referencedButLater.isEmpty(), String.format(
+                "the segment holding the compacted root %s (%s) references %d segment(s) that "
+                        + "are only persisted after it: %s",
+                rootSegment, rootSegment.getGcGeneration(), referencedButLater.size(),
+                referencedButLater.stream()
+                        .map(id -> id + " " + id.getGcGeneration())
+                        .collect(Collectors.toList())));
+    }
+
+    private static Set<UUID> referencedSegmentIds(Segment segment) {
+        Set<UUID> referenced = new HashSet<>();
+        for (int i = 0; i < segment.getReferencedSegmentIdCount(); i++) {
+            referenced.add(segment.getReferencedSegmentId(i));
+        }
+        return referenced;
+    }
+
+    /**
+     * Records the order in which segments reach the underlying store. The recording list
+     * belongs to the enclosing instance because {@link MemoryStore#MemoryStore()} already
+     * flushes a segment, before any field of this subclass could be initialised.
+     */
+    private class RecordingStore extends MemoryStore {
+
+        RecordingStore() throws IOException {
+            super();
+        }
+
+        @Override
+        public void writeSegment(SegmentId id, byte[] data, int offset, int length) throws IOException {
+            persisted.add(id);
+            super.writeSegment(id, data, offset, length);
+        }
+    }
+}