You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2014/12/16 15:56:08 UTC
svn commit: r1645959 - in /jackrabbit/oak/branches/1.0/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/
test/java/org/apache/jackrabbit/oak/plugins/segment/
test/java/org/apache/jackrabbit/oak/plugins/segment/file/
Author: mduerig
Date: Tue Dec 16 14:56:08 2014
New Revision: 1645959
URL: http://svn.apache.org/r1645959
Log:
OAK-2192 Concurrent commit during compaction results in mixed segments
Missing files from last commit
Added:
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategy.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategyMBean.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/DefaultCompactionStrategyMBean.java
jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java
jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java
jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java
Added: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategy.java?rev=1645959&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategy.java (added)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategy.java Tue Dec 16 14:56:08 2014
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment.compaction;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.System.currentTimeMillis;
+
+import java.util.concurrent.Callable;
+
+import javax.annotation.CheckForNull;
+import javax.annotation.Nonnull;
+
+import org.apache.jackrabbit.oak.plugins.segment.CompactionMap;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
+
+public class CompactionStrategy {
+
+ public enum CleanupType {
+
+ /**
+ * {@code CLEAN_ALL} <em>must</em> be used in conjunction with {@code cloneBinaries}
+ * otherwise segments can go away ({@code SegmentNotFoundException})
+ * <p>
+ * Pros: best compaction results
+ * <p>
+ * Cons: larger repo size <em>during</em> compaction (2x). High chances that a currently
+ * running diff (e.g. observation) fails with {@code SegmentNotFoundException}.
+ */
+ CLEAN_ALL,
+
+ CLEAN_NONE,
+
+ /**
+ * {@code CLEAN_OLD} with {@code cloneBinaries}
+ * <p>
+ * Pros: better compaction results
+ * <p>
+ * Cons: larger repo size {@code during} compaction (2x). {@code SegmentNotFoundException}
+ * with insufficiently large values for {@code olderThan}.
+ * <p>
+ * {@code CLEAN_OLD} without {@code cloneBinaries}
+ * <p>
+ * Pros: weakest compaction results, smaller size during compaction (1x + size of
+ * data-segments).
+ * <p>
+ * Cons: {@code SegmentNotFoundException} with insufficiently large values for
+ * {@code olderThan}.
+ */
+ CLEAN_OLD
+ }
+
+ public static final boolean PAUSE_DEFAULT = true;
+
+ public static final boolean CLONE_BINARIES_DEFAULT = false;
+
+ public static final CleanupType CLEANUP_DEFAULT = CleanupType.CLEAN_OLD;
+
+ public static final long TIMESTAMP_DEFAULT = 1000 * 60 * 5;
+
+ public static final byte MEMORY_THRESHOLD_DEFAULT = 5;
+
+ private boolean paused;
+
+ private boolean cloneBinaries;
+
+ @Nonnull
+ private CleanupType cleanupType;
+
+ /**
+ * anything that has a lifetime bigger than this will be removed. a value of
+ * 0 (or very small) acts like a CLEANUP.NONE, a value of -1 (or negative)
+ * acts like a CLEANUP.ALL
+ *
+ */
+ private long olderThan;
+
+ private byte memoryThreshold = MEMORY_THRESHOLD_DEFAULT;
+
+ private CompactionMap compactionMap;
+
+ private long compactionStart = currentTimeMillis();
+
+ public CompactionStrategy(boolean paused,
+ boolean cloneBinaries, @Nonnull CleanupType cleanupType, long olderThan, byte memoryThreshold) {
+ checkArgument(olderThan >= 0);
+ this.paused = paused;
+ this.cloneBinaries = cloneBinaries;
+ this.cleanupType = checkNotNull(cleanupType);
+ this.olderThan = olderThan;
+ this.memoryThreshold = memoryThreshold;
+ }
+
+ public boolean canRemove(SegmentId id) {
+ switch (cleanupType) {
+ case CLEAN_ALL:
+ return true;
+ case CLEAN_NONE:
+ return false;
+ case CLEAN_OLD:
+ return compactionStart - id.getCreationTime() > olderThan;
+ }
+ return false;
+ }
+
+ public boolean cloneBinaries() {
+ return cloneBinaries;
+ }
+
+ public boolean isPaused() {
+ return paused;
+ }
+
+ public void setPaused(boolean paused) {
+ this.paused = paused;
+ }
+
+ public void setCloneBinaries(boolean cloneBinaries) {
+ this.cloneBinaries = cloneBinaries;
+ }
+
+ public void setCleanupType(@Nonnull CleanupType cleanupType) {
+ this.cleanupType = checkNotNull(cleanupType);
+ }
+
+ public void setOlderThan(long olderThan) {
+ checkArgument(olderThan >= 0);
+ this.olderThan = olderThan;
+ }
+
+ public void setCompactionMap(@Nonnull CompactionMap compactionMap) {
+ this.compactionMap = checkNotNull(compactionMap);
+ }
+
+ String getCleanupType() {
+ return cleanupType.toString();
+ }
+
+ long getOlderThan() {
+ return olderThan;
+ }
+
+ @CheckForNull
+ public CompactionMap getCompactionMap() {
+ return this.compactionMap;
+ }
+
+ @Override
+ public String toString() {
+ return "DefaultCompactionStrategy [pauseCompaction=" + paused
+ + ", cloneBinaries=" + cloneBinaries + ", cleanup=" + cleanupType
+ + ", olderThan=" + olderThan + ']';
+ }
+
+ public void setCompactionStart(long ms) {
+ this.compactionStart = ms;
+ }
+
+ public byte getMemoryThreshold() {
+ return memoryThreshold;
+ }
+
+ public void setMemoryThreshold(byte memoryThreshold) {
+ this.memoryThreshold = memoryThreshold;
+ }
+
+ public boolean compacted(@Nonnull Callable<Boolean> setHead) throws Exception {
+ return checkNotNull(setHead).call();
+ }
+}
Added: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategyMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategyMBean.java?rev=1645959&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategyMBean.java (added)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/CompactionStrategyMBean.java Tue Dec 16 14:56:08 2014
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment.compaction;
+
+public interface CompactionStrategyMBean {
+
+ String TYPE = "CompactionStrategy";
+
+ boolean isCloneBinaries();
+
+ void setCloneBinaries(boolean cloneBinaries);
+
+ boolean isPausedCompaction();
+
+ void setPausedCompaction(boolean pausedCompaction);
+
+ String getCleanupStrategy();
+
+ void setCleanupStrategy(String cleanup);
+
+ long getOlderThan();
+
+ void setOlderThan(long olderThan);
+
+ byte getMemoryThreshold();
+
+ void setMemoryThreshold(byte memory);
+
+ String getCompactionMapStats();
+}
Added: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/DefaultCompactionStrategyMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/DefaultCompactionStrategyMBean.java?rev=1645959&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/DefaultCompactionStrategyMBean.java (added)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/DefaultCompactionStrategyMBean.java Tue Dec 16 14:56:08 2014
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment.compaction;
+
+import org.apache.jackrabbit.oak.plugins.segment.CompactionMap;
+import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType;
+
+public class DefaultCompactionStrategyMBean implements CompactionStrategyMBean {
+
+ private final CompactionStrategy strategy;
+
+ public DefaultCompactionStrategyMBean(CompactionStrategy strategy) {
+ this.strategy = strategy;
+ }
+
+ @Override
+ public boolean isCloneBinaries() {
+ return strategy.cloneBinaries();
+ }
+
+ @Override
+ public void setCloneBinaries(boolean cloneBinaries) {
+ strategy.setCloneBinaries(cloneBinaries);
+ }
+
+ @Override
+ public boolean isPausedCompaction() {
+ return strategy.isPaused();
+ }
+
+ @Override
+ public void setPausedCompaction(boolean pausedCompaction) {
+ strategy.setPaused(pausedCompaction);
+ }
+
+ @Override
+ public String getCleanupStrategy() {
+ return strategy.getCleanupType();
+ }
+
+ @Override
+ public void setCleanupStrategy(String cleanup) {
+ strategy.setCleanupType(CleanupType.valueOf(cleanup));
+ }
+
+ @Override
+ public long getOlderThan() {
+ return strategy.getOlderThan();
+ }
+
+ @Override
+ public void setOlderThan(long olderThan) {
+ strategy.setOlderThan(olderThan);
+ }
+
+ @Override
+ public byte getMemoryThreshold() {
+ return strategy.getMemoryThreshold();
+ }
+
+ @Override
+ public void setMemoryThreshold(byte memory) {
+ strategy.setMemoryThreshold(memory);
+ }
+
+ @Override
+ public String getCompactionMapStats() {
+ CompactionMap cm = strategy.getCompactionMap();
+ if (cm != null) {
+ return cm.getCompactionStats();
+ }
+ return "";
+ }
+}
Added: jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java?rev=1645959&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java (added)
+++ jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java Tue Dec 16 14:56:08 2014
@@ -0,0 +1,282 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.apache.commons.io.FileUtils.deleteDirectory;
+import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType.CLEAN_NONE;
+import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType.CLEAN_OLD;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import com.google.common.io.ByteStreams;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class CompactionAndCleanupTest {
+
+ private File directory;
+
+ @Before
+ public void setUp() throws IOException {
+ directory = File.createTempFile(
+ "FileStoreTest", "dir", new File("target"));
+ directory.delete();
+ directory.mkdir();
+ }
+
+ @Test
+ @Ignore("OAK-2045")
+ public void compactionAndWeakReferenceMagic() throws Exception {
+ final int MB = 1024 * 1024;
+ final int blobSize = 5 * MB;
+
+ final int dataNodes = 10000;
+
+ // really long time span, no binary cloning
+ CompactionStrategy custom = new CompactionStrategy(false,
+ false, CLEAN_OLD, TimeUnit.HOURS.toMillis(1), (byte) 0);
+
+ FileStore fileStore = new FileStore(directory, 1);
+ SegmentNodeStore nodeStore = new SegmentNodeStore(fileStore);
+ fileStore.setCompactionStrategy(custom);
+
+ // 1a. Create a bunch of data
+ NodeBuilder extra = nodeStore.getRoot().builder();
+ NodeBuilder content = extra.child("content");
+ for (int i = 0; i < dataNodes; i++) {
+ NodeBuilder c = content.child("c" + i);
+ for (int j = 0; j < 1000; j++) {
+ c.setProperty("p" + i, "v" + i);
+ }
+ }
+ nodeStore.merge(extra, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ // ----
+
+ final long dataSize = fileStore.size();
+ System.out.printf("File store dataSize %s%n",
+ byteCountToDisplaySize(dataSize));
+
+ // 1. Create a property with 5 MB blob
+ NodeBuilder builder = nodeStore.getRoot().builder();
+ builder.setProperty("a1", createBlob(nodeStore, blobSize));
+ builder.setProperty("b", "foo");
+
+ // Keep a reference to this nodeState to simulate long
+ // running session
+ NodeState ns1 = nodeStore.merge(builder, EmptyHook.INSTANCE,
+ CommitInfo.EMPTY);
+
+ System.out.printf("File store pre removal %s expecting %s %n",
+ byteCountToDisplaySize(fileStore.size()),
+ byteCountToDisplaySize(blobSize + dataSize));
+ assertEquals(mb(blobSize + dataSize), mb(fileStore.size()));
+
+ // 2. Now remove the property
+ builder = nodeStore.getRoot().builder();
+ builder.removeProperty("a1");
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // Size remains same
+ System.out.printf("File store pre compaction %s expecting %s%n",
+ byteCountToDisplaySize(fileStore.size()),
+ byteCountToDisplaySize(blobSize + dataSize));
+ assertEquals(mb(blobSize + dataSize), mb(fileStore.size()));
+
+ // 3. Compact
+ assertTrue(fileStore.maybeCompact(false));
+ // fileStore.cleanup();
+
+ // Size still remains same
+ System.out.printf("File store post compaction %s expecting %s%n",
+ byteCountToDisplaySize(fileStore.size()),
+ byteCountToDisplaySize(blobSize + dataSize));
+ assertEquals("File store post compaction size",
+ mb(blobSize + dataSize), mb(fileStore.size()));
+
+ // 4. Add some more property to flush the current TarWriter
+ builder = nodeStore.getRoot().builder();
+ builder.setProperty("a2", createBlob(nodeStore, blobSize));
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // Size is double
+ System.out.printf("File store pre cleanup %s expecting %s%n",
+ byteCountToDisplaySize(fileStore.size()),
+ byteCountToDisplaySize(2 * blobSize + dataSize));
+ assertEquals(mb(2 * blobSize + dataSize), mb(fileStore.size()));
+
+ // 5. Cleanup
+ assertTrue(fileStore.maybeCompact(false));
+ fileStore.cleanup();
+
+ System.out.printf(
+ "File store post cleanup %s expecting between [%s,%s]%n",
+ byteCountToDisplaySize(fileStore.size()),
+ byteCountToDisplaySize(blobSize + dataSize),
+ byteCountToDisplaySize(blobSize + 2 * dataSize));
+
+ // 0 data size: fileStore.size() == blobSize
+ // >0 data size: fileStore.size() in [blobSize + dataSize, blobSize +
+ // 2xdataSize]
+ assertTrue(mb(fileStore.size()) >= mb(blobSize + dataSize)
+ && mb(fileStore.size()) <= mb(blobSize + 2 * dataSize));
+
+ // refresh the ts ref, to simulate a long wait time
+ custom.setOlderThan(0);
+ assertFalse(fileStore.maybeCompact(false));
+
+ // no data loss happened
+ byte[] blob = ByteStreams.toByteArray(nodeStore.getRoot()
+ .getProperty("a2").getValue(Type.BINARY).getNewStream());
+ assertEquals(blobSize, blob.length);
+ }
+
+ @After
+ public void cleanDir() throws IOException {
+ deleteDirectory(directory);
+ }
+
+ private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
+ byte[] data = new byte[size];
+ new Random().nextBytes(data);
+ return nodeStore.createBlob(new ByteArrayInputStream(data));
+ }
+
+ private static long mb(long size){
+ return size / (1024 * 1024);
+ }
+
+ @Test
+ public void testMixedSegments() throws Exception {
+ FileStore store = new FileStore(directory, 2, false);
+ final SegmentNodeStore nodeStore = new SegmentNodeStore(store);
+ store.setCompactionStrategy(new CompactionStrategy(true, false, CLEAN_NONE, 0, (byte) 5) {
+ @Override
+ public boolean compacted(Callable<Boolean> setHead) throws Exception {
+ return nodeStore.locked(setHead);
+ }
+ });
+
+ NodeBuilder root = nodeStore.getRoot().builder();
+ createNodes(root.setChildNode("test"), 10, 3);
+ nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ final Set<UUID> beforeSegments = new HashSet<UUID>();
+ collectSegments(store.getHead(), beforeSegments);
+
+ final AtomicReference<Boolean> run = new AtomicReference<Boolean>(true);
+ final List<Integer> failedCommits = newArrayList();
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ for (int k = 0; run.get(); k++) {
+ try {
+ NodeBuilder root = nodeStore.getRoot().builder();
+ root.setChildNode("b" + k);
+ nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ Thread.sleep(5);
+ } catch (CommitFailedException e) {
+ failedCommits.add(k);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ break;
+ }
+ }
+ }
+ });
+ t.start();
+
+ store.compact();
+ run.set(false);
+ t.join();
+
+ assertTrue(failedCommits.isEmpty());
+
+ Set<UUID> afterSegments = new HashSet<UUID>();
+ collectSegments(store.getHead(), afterSegments);
+ try {
+ for (UUID u : beforeSegments) {
+ assertFalse("Mixed segments found: " + u, afterSegments.contains(u));
+ }
+ } finally {
+ store.close();
+ }
+ }
+
+ private static void collectSegments(SegmentNodeState s, Set<UUID> segmentIds) {
+ SegmentId sid = s.getRecordId().getSegmentId();
+ UUID id = new UUID(sid.getMostSignificantBits(),
+ sid.getLeastSignificantBits());
+ segmentIds.add(id);
+ for (ChildNodeEntry cne : s.getChildNodeEntries()) {
+ collectSegments((SegmentNodeState) cne.getNodeState(), segmentIds);
+ }
+ for (PropertyState propertyState : s.getProperties()) {
+ sid = ((SegmentPropertyState) propertyState).getRecordId().getSegmentId();
+ id = new UUID(sid.getMostSignificantBits(),
+ sid.getLeastSignificantBits());
+ segmentIds.add(id);
+ }
+ }
+
+ private static void createNodes(NodeBuilder builder, int count, int depth) {
+ if (depth > 0) {
+ for (int k = 0; k < count; k++) {
+ NodeBuilder child = builder.setChildNode("node" + k);
+ createProperties(child, count);
+ createNodes(child, count, depth - 1);
+ }
+ }
+ }
+
+ private static void createProperties(NodeBuilder builder, int count) {
+ for (int k = 0; k < count; k++) {
+ builder.setProperty("property-" + UUID.randomUUID().toString(), "value-" + UUID.randomUUID().toString());
+ }
+ }
+}
Added: jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java?rev=1645959&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java (added)
+++ jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java Tue Dec 16 14:56:08 2014
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static org.apache.commons.io.FileUtils.deleteDirectory;
+import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType.CLEAN_OLD;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class HeavyWriteIT {
+
+ private File directory;
+
+ @Before
+ public void setUp() throws IOException {
+ directory = File.createTempFile(
+ "FileStoreTest", "dir", new File("target"));
+ directory.delete();
+ directory.mkdir();
+ }
+
+ @After
+ public void cleanDir() throws IOException {
+ deleteDirectory(directory);
+ }
+
+ @Test
+ public void heavyWrite() throws IOException, CommitFailedException, InterruptedException {
+ final FileStore store = new FileStore(directory, 128, false);
+ store.setCompactionStrategy(new CompactionStrategy(false, false,
+ CLEAN_OLD, 30000, (byte) 0));
+ final SegmentNodeStore nodeStore = new SegmentNodeStore(store);
+
+ int writes = 100;
+ final AtomicBoolean run = new AtomicBoolean(true);
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ for (int k = 1; run.get(); k++) {
+ store.gc();
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ }
+ });
+ thread.start();
+
+ try {
+ for (int k = 1; k<=writes; k++) {
+ NodeBuilder root = nodeStore.getRoot().builder();
+ NodeBuilder test = root.setChildNode("test");
+ createNodes(nodeStore, test, 10, 2);
+ nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ root = nodeStore.getRoot().builder();
+ root.getChildNode("test").remove();
+ nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+
+ } finally {
+ run.set(false);
+ thread.join();
+ store.close();
+ }
+ }
+
+ private static void createNodes(NodeStore nodeStore, NodeBuilder builder, int count, int depth) throws IOException {
+ if (depth > 0) {
+ for (int k = 0; k < count; k++) {
+ NodeBuilder child = builder.setChildNode("node" + k);
+ createProperties(nodeStore, child, count);
+ createNodes(nodeStore, child, count, depth - 1);
+ }
+ }
+ }
+
+ private static void createProperties(NodeStore nodeStore, NodeBuilder builder, int count) throws IOException {
+ for (int k = 0; k < count; k++) {
+ builder.setProperty("property-" + k, createBlob(nodeStore, 100000));
+ }
+ }
+
+ private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
+ byte[] data = new byte[size];
+ new Random().nextBytes(data);
+ return nodeStore.createBlob(new ByteArrayInputStream(data));
+ }
+
+}
Added: jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java?rev=1645959&view=auto
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java (added)
+++ jackrabbit/oak/branches/1.0/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java Tue Dec 16 14:56:08 2014
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment.file;
+
+import static org.apache.commons.io.FileUtils.deleteDirectory;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CompactionEstimatorTest {
+
+ private File directory;
+
+ @Before
+ public void setUp() throws IOException {
+ directory = File.createTempFile(
+ "FileStoreTest", "dir", new File("target"));
+ directory.delete();
+ directory.mkdir();
+ }
+
+ @After
+ public void cleanDir() throws IOException {
+ deleteDirectory(directory);
+ }
+
+ @Test
+ public void testGainEstimator() throws Exception {
+ final int MB = 1024 * 1024;
+ final int blobSize = 2 * MB;
+
+ FileStore fileStore = new FileStore(directory, 2, false);
+ SegmentNodeStore nodeStore = new SegmentNodeStore(fileStore);
+
+ // 1. Create some blob properties
+ NodeBuilder builder = nodeStore.getRoot().builder();
+
+ NodeBuilder c1 = builder.child("c1");
+ c1.setProperty("a", createBlob(nodeStore, blobSize));
+ c1.setProperty("b", "foo");
+
+ NodeBuilder c2 = builder.child("c2");
+ c2.setProperty("a", createBlob(nodeStore, blobSize));
+ c2.setProperty("b", "foo");
+
+ NodeBuilder c3 = builder.child("c3");
+ c3.setProperty("a", createBlob(nodeStore, blobSize));
+ c3.setProperty("b", "foo");
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // 2. Now remove the property
+ builder = nodeStore.getRoot().builder();
+ builder.child("c1").remove();
+ builder.child("c2").remove();
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ fileStore.flush();
+ try {
+ // should be at 66%
+ assertTrue(fileStore.estimateCompactionGain()
+ .estimateCompactionGain() > 60);
+ } finally {
+ fileStore.close();
+ }
+ }
+
+ private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
+ byte[] data = new byte[size];
+ new Random().nextBytes(data);
+ return nodeStore.createBlob(new ByteArrayInputStream(data));
+ }
+
+}