You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2014/12/15 13:36:36 UTC
svn commit: r1645637 [2/2] - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/
main/java/org/apache/jackrabbit/oak/plugins/segment/compaction/
main/java/org/apache/jackrabbit/oak/plugins/segment/file/ test/java...
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java?rev=1645637&r1=1645636&r2=1645637&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java Mon Dec 15 12:36:36 2014
@@ -18,8 +18,8 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.collect.Maps.newHashMap;
import static junit.framework.Assert.assertTrue;
-import static org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ALIGN_BITS;
import static org.apache.jackrabbit.oak.plugins.segment.Segment.MAX_SEGMENT_SIZE;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ALIGN_BITS;
import static org.junit.Assert.assertFalse;
import java.util.Map;
@@ -39,8 +39,8 @@ public class CompactionMapTest {
System.gc();
System.out.println((runtime.totalMemory() - runtime.freeMemory()) / (1024 * 1024));
- CompactionMap map = new CompactionMap(100000);
SegmentTracker factory = new MemoryStore().getTracker();
+ CompactionMap map = new CompactionMap(100000, factory);
for (int i = 0; i < 1000000; i++) {
if (i % 1000 == 0) {
System.gc();
@@ -65,7 +65,7 @@ public class CompactionMapTest {
Random r = new Random(seed);
SegmentTracker factory = new MemoryStore().getTracker();
- CompactionMap map = new CompactionMap(r.nextInt(maxSegments / 2));
+ CompactionMap map = new CompactionMap(r.nextInt(maxSegments / 2), factory);
Map<RecordId, RecordId> entries = newHashMap();
int segments = r.nextInt(maxSegments);
@@ -101,7 +101,7 @@ public class CompactionMapTest {
* Returns a new valid record offset, between {@code a} and {@code b},
* exclusive.
*/
- private int newValidOffset(Random random, int a, int b) {
+ private static int newValidOffset(Random random, int a, int b) {
int p = (a >> RECORD_ALIGN_BITS) + 1;
int q = (b >> RECORD_ALIGN_BITS);
return (p + random.nextInt(q - p)) << RECORD_ALIGN_BITS;
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java?rev=1645637&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java Mon Dec 15 12:36:36 2014
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static org.apache.commons.io.FileUtils.deleteDirectory;
+import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType.CLEAN_OLD;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class HeavyWriteIT {
+
+ private File directory;
+
+ @Before
+ public void setUp() throws IOException {
+ directory = File.createTempFile(
+ "FileStoreTest", "dir", new File("target"));
+ directory.delete();
+ directory.mkdir();
+ }
+
+ @After
+ public void cleanDir() throws IOException {
+ deleteDirectory(directory);
+ }
+
+ @Test
+ public void heavyWrite() throws IOException, CommitFailedException, InterruptedException {
+ final FileStore store = new FileStore(directory, 128, false);
+ store.setCompactionStrategy(new CompactionStrategy(false, false,
+ CLEAN_OLD, 30000, (byte) 0));
+ final SegmentNodeStore nodeStore = new SegmentNodeStore(store);
+
+ int writes = 100;
+ final AtomicBoolean run = new AtomicBoolean(true);
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ for (int k = 1; run.get(); k++) {
+ store.gc();
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ }
+ });
+ thread.start();
+
+ try {
+ for (int k = 1; k<=writes; k++) {
+ NodeBuilder root = nodeStore.getRoot().builder();
+ NodeBuilder test = root.setChildNode("test");
+ createNodes(nodeStore, test, 10, 2);
+ nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ root = nodeStore.getRoot().builder();
+ root.getChildNode("test").remove();
+ nodeStore.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+
+ } finally {
+ run.set(false);
+ thread.join();
+ store.close();
+ }
+ }
+
+ private static void createNodes(NodeStore nodeStore, NodeBuilder builder, int count, int depth) throws IOException {
+ if (depth > 0) {
+ for (int k = 0; k < count; k++) {
+ NodeBuilder child = builder.setChildNode("node" + k);
+ createProperties(nodeStore, child, count);
+ createNodes(nodeStore, child, count, depth - 1);
+ }
+ }
+ }
+
+ private static void createProperties(NodeStore nodeStore, NodeBuilder builder, int count) throws IOException {
+ for (int k = 0; k < count; k++) {
+ builder.setProperty("property-" + k, createBlob(nodeStore, 100000));
+ }
+ }
+
+ private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
+ byte[] data = new byte[size];
+ new Random().nextBytes(data);
+ return nodeStore.createBlob(new ByteArrayInputStream(data));
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java?rev=1645637&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java Mon Dec 15 12:36:36 2014
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment.file;
+
+import static org.apache.commons.io.FileUtils.deleteDirectory;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class CompactionEstimatorTest {
+
+ private File directory;
+
+ @Before
+ public void setUp() throws IOException {
+ directory = File.createTempFile(
+ "FileStoreTest", "dir", new File("target"));
+ directory.delete();
+ directory.mkdir();
+ }
+
+ @After
+ public void cleanDir() throws IOException {
+ deleteDirectory(directory);
+ }
+
+ @Test
+ public void testGainEstimator() throws Exception {
+ final int MB = 1024 * 1024;
+ final int blobSize = 2 * MB;
+
+ FileStore fileStore = new FileStore(directory, 2, false);
+ SegmentNodeStore nodeStore = new SegmentNodeStore(fileStore);
+
+ // 1. Create some blob properties
+ NodeBuilder builder = nodeStore.getRoot().builder();
+
+ NodeBuilder c1 = builder.child("c1");
+ c1.setProperty("a", createBlob(nodeStore, blobSize));
+ c1.setProperty("b", "foo");
+
+ NodeBuilder c2 = builder.child("c2");
+ c2.setProperty("a", createBlob(nodeStore, blobSize));
+ c2.setProperty("b", "foo");
+
+ NodeBuilder c3 = builder.child("c3");
+ c3.setProperty("a", createBlob(nodeStore, blobSize));
+ c3.setProperty("b", "foo");
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // 2. Now remove the property
+ builder = nodeStore.getRoot().builder();
+ builder.child("c1").remove();
+ builder.child("c2").remove();
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ fileStore.flush();
+ try {
+ // should be at 66%
+ assertTrue(fileStore.estimateCompactionGain()
+ .estimateCompactionGain() > 60);
+ } finally {
+ fileStore.close();
+ }
+ }
+
+ private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
+ byte[] data = new byte[size];
+ new Random().nextBytes(data);
+ return nodeStore.createBlob(new ByteArrayInputStream(data));
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java?rev=1645637&r1=1645636&r2=1645637&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreTest.java Mon Dec 15 12:36:36 2014
@@ -110,6 +110,10 @@ public class FileStoreTest {
SegmentNodeState head = builder.getNodeState();
assertTrue(store.setHead(base, head));
assertEquals("bar", store.getHead().getString("foo"));
+
+ Compactor compactor = new Compactor(writer);
+ SegmentNodeState compacted =
+ compactor.compact(EmptyNodeState.EMPTY_NODE, head);
store.close();
// First simulate the case where during compaction a reference to the
@@ -117,9 +121,6 @@ public class FileStoreTest {
store = new FileStore(directory, 1, false);
head = store.getHead();
assertTrue(store.size() > largeBinarySize);
- Compactor compactor = new Compactor(writer);
- SegmentNodeState compacted =
- compactor.compact(EmptyNodeState.EMPTY_NODE, head);
builder = head.builder();
builder.setChildNode("old", head); // reference to pre-compacted state
builder.getNodeState();