You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2016/10/31 10:12:37 UTC
svn commit: r1767237 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/
oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/
oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/
oak-run/src/main/jav...
Author: tomekr
Date: Mon Oct 31 10:12:36 2016
New Revision: 1767237
URL: http://svn.apache.org/viewvc?rev=1767237&view=rev
Log:
OAK-5035: Implement mini-benchmark for the PersistentCache
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/PersistentCacheTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1767237&r1=1767236&r2=1767237&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Mon Oct 31 10:12:36 2016
@@ -837,6 +837,10 @@ public final class DocumentNodeStore
return diffCache.getStats();
}
+ Cache<PathRev, DocumentNodeState> getNodeCache() {
+ return nodeCache;
+ }
+
/**
* Returns the journal entry that will be stored in the journal with the
* next background updated.
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java?rev=1767237&r1=1767236&r2=1767237&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java Mon Oct 31 10:12:36 2016
@@ -416,7 +416,8 @@ public class BenchmarkRunner {
new SyncAllExternalUsersTest(numberOfUsers.value(options), numberOfGroups.value(options), expiration.value(options), dynamicMembership.value(options), autoMembership.values(options)),
new SyncExternalUsersTest(numberOfUsers.value(options), numberOfGroups.value(options), expiration.value(options), dynamicMembership.value(options), autoMembership.values(options), batchSize.value(options)),
new HybridIndexTest(base.value(options), statsProvider),
- new BundlingNodeTest()
+ new BundlingNodeTest(),
+ new PersistentCacheTest()
};
Set<String> argset = Sets.newHashSet(nonOption.values(options));
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/PersistentCacheTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/PersistentCacheTest.java?rev=1767237&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/PersistentCacheTest.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/PersistentCacheTest.java Mon Oct 31 10:12:36 2016
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.benchmark;
+
+import com.google.common.cache.Cache;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.fixture.OakFixture;
+import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
+import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeState;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper;
+import org.apache.jackrabbit.oak.plugins.document.PathRev;
+import org.apache.jackrabbit.oak.plugins.document.Revision;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+
+import javax.jcr.Repository;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class PersistentCacheTest extends AbstractTest {
+
+ private static final int ITEMS_TO_ADD = Integer.getInteger("items", 10000);
+
+ private static final String CACHE_OPTIONS = System.getProperty("cacheOptions", "size=100,+compact,-async");
+
+ private Cache<PathRev, DocumentNodeState> nodesCache;
+
+ private DocumentNodeStore dns;
+
+ private AtomicLong timestamp = new AtomicLong(1000);
+
+ @Override
+ protected Repository[] createRepository(RepositoryFixture fixture) throws Exception {
+ if (fixture instanceof OakRepositoryFixture) {
+ OakFixture oakFixture = ((OakRepositoryFixture) fixture).getOakFixture();
+ if (oakFixture instanceof OakFixture.MongoFixture) {
+ OakFixture.MongoFixture mongoFixture = (OakFixture.MongoFixture) oakFixture;
+ DocumentMK.Builder builder = mongoFixture.getBuilder(1);
+ builder.setPersistentCache("target/persistentCache,time," + CACHE_OPTIONS);
+ dns = builder.getNodeStore();
+ nodesCache = DocumentNodeStoreHelper.getNodesCache(dns);
+ Oak[] cluster = mongoFixture.setUpCluster(new DocumentMK.Builder[] {builder}, StatisticsProvider.NOOP);
+ return new Repository[] { new Jcr(cluster[0]).createRepository() };
+ }
+ }
+ throw new IllegalArgumentException("Fixture " + fixture + " not supported for this benchmark.");
+ }
+
+ @Override
+ protected void beforeSuite() throws Exception {
+ }
+
+ @Override
+ protected void beforeTest() throws Exception {
+ }
+
+ @Override
+ protected void runTest() throws Exception {
+ for (int i = 0; i < ITEMS_TO_ADD; i++) {
+ PathRev key = PathRev.fromString("/" + timestamp.getAndIncrement() + "@" + new Revision(timestamp.getAndIncrement(), 0, 0));
+ nodesCache.put(key, dns.getRoot());
+ nodesCache.getIfPresent(key); // read, so the entry is marked as used
+ }
+ }
+
+ @Override
+ protected void afterTest() throws Exception {
+ }
+}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java?rev=1767237&r1=1767236&r2=1767237&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/fixture/OakFixture.java Mon Oct 31 10:12:36 2016
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.fixtur
import java.io.File;
import java.lang.management.ManagementFactory;
+import java.net.UnknownHostException;
import javax.sql.DataSource;
@@ -157,83 +158,7 @@ public abstract class OakFixture {
final boolean dropDBAfterTest, final long cacheSize,
final boolean useDataStore,
final File base, final int dsCacheInMB) {
- return new OakFixture(name) {
- private DocumentMK[] kernels;
- private BlobStoreFixture blobStoreFixture;
-
- @Override
- public Oak getOak(int clusterId) throws Exception {
- MongoConnection mongo = new MongoConnection(uri);
- DocumentMK.Builder mkBuilder = new DocumentMK.Builder().
- setMongoDB(mongo.getDB()).
- memoryCacheSize(cacheSize).
- //TODO Persistent cache should be removed in teardown
- setPersistentCache("target/persistentCache,time").
- setClusterId(clusterId).
- setLogging(false);
- setupBlobStore(mkBuilder, StatisticsProvider.NOOP);
- DocumentMK dmk = mkBuilder.open();
- return newOak(dmk.getNodeStore());
- }
-
- @Override
- public Oak[] setUpCluster(int n, StatisticsProvider statsProvider) throws Exception {
- Oak[] cluster = new Oak[n];
- kernels = new DocumentMK[cluster.length];
- for (int i = 0; i < cluster.length; i++) {
- MongoConnection mongo = new MongoConnection(uri);
- DocumentMK.Builder mkBuilder = new DocumentMK.Builder().
- setStatisticsProvider(statsProvider).
- setMongoDB(mongo.getDB()).
- memoryCacheSize(cacheSize).
- setPersistentCache("target/persistentCache,time").
- setClusterId(i + 1).
- setLogging(false);
- setupBlobStore(mkBuilder, statsProvider);
- kernels[i] = mkBuilder.open();
- cluster[i] = newOak(kernels[i].getNodeStore());
- }
- return cluster;
- }
-
- @Override
- public void tearDownCluster() {
- for (DocumentMK kernel : kernels) {
- kernel.dispose();
- }
- if (dropDBAfterTest) {
- try {
- MongoConnection mongo =
- new MongoConnection(uri);
- mongo.getDB().dropDatabase();
- mongo.close();
- if(blobStoreFixture != null){
- blobStoreFixture.tearDown();
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- private void setupBlobStore(DocumentMK.Builder mkBuilder, StatisticsProvider statsProvider) {
- initializeBlobStoreFixture(statsProvider);
- if (blobStoreFixture != null) {
- mkBuilder.setBlobStore(blobStoreFixture.setUp());
- }
- }
-
- private void initializeBlobStoreFixture(StatisticsProvider statsProvider) {
- if (blobStoreFixture != null){
- return;
- }
-
- if (useDataStore) {
- blobStoreFixture =
- BlobStoreFixture.create(base, true, dsCacheInMB, statsProvider);
- }
- }
- };
+ return new MongoFixture(name, uri, dropDBAfterTest, cacheSize, useDataStore, base, dsCacheInMB);
}
public static OakFixture getRDB(final String name, final String jdbcuri, final String jdbcuser, final String jdbcpasswd,
@@ -354,6 +279,114 @@ public abstract class OakFixture {
return new SegmentTarFixture(name, base, maxFileSizeMB, cacheSizeMB, memoryMapping, useBlobStore, dsCacheInMB);
}
+ public static class MongoFixture extends OakFixture {
+
+ private final String uri;
+
+ private final boolean dropDBAfterTest;
+
+ private final long cacheSize;
+
+ private final boolean useDataStore;
+
+ private final File base;
+
+ private final int dsCacheInMB;
+
+ private DocumentMK[] kernels;
+ private BlobStoreFixture blobStoreFixture;
+
+ public MongoFixture(final String name, final String uri,
+ final boolean dropDBAfterTest, final long cacheSize,
+ final boolean useDataStore,
+ final File base, final int dsCacheInMB) {
+ super(name);
+ this.uri = uri;
+ this.dropDBAfterTest = dropDBAfterTest;
+ this.cacheSize = cacheSize;
+ this.useDataStore = useDataStore;
+ this.base = base;
+ this.dsCacheInMB = dsCacheInMB;
+ }
+
+ public DocumentMK.Builder getBuilder(int clusterId) throws UnknownHostException {
+ MongoConnection mongo = new MongoConnection(uri);
+ DocumentMK.Builder mkBuilder = new DocumentMK.Builder().
+ setMongoDB(mongo.getDB()).
+ memoryCacheSize(cacheSize).
+ //TODO Persistent cache should be removed in teardown
+ setPersistentCache("target/persistentCache,time").
+ setClusterId(clusterId).
+ setLogging(false);
+ setupBlobStore(mkBuilder, StatisticsProvider.NOOP);
+ return mkBuilder;
+ }
+
+ @Override
+ public Oak getOak(int clusterId) throws Exception {
+ DocumentMK dmk = getBuilder(clusterId).open();
+ return newOak(dmk.getNodeStore());
+ }
+
+ public Oak[] setUpCluster(DocumentMK.Builder[] builders, StatisticsProvider statsProvider) throws Exception {
+ Oak[] cluster = new Oak[builders.length];
+ kernels = new DocumentMK[cluster.length];
+ for (int i = 0; i < cluster.length; i++) {
+ kernels[i] = builders[i].open();
+ cluster[i] = newOak(kernels[i].getNodeStore());
+ }
+ return cluster;
+ }
+
+ @Override
+ public Oak[] setUpCluster(int n, StatisticsProvider statsProvider) throws Exception {
+ DocumentMK.Builder[] builders = new DocumentMK.Builder[n];
+ for (int i = 0; i < n; i++) {
+ builders[i] = getBuilder(i + 1);
+ }
+ return setUpCluster(builders, statsProvider);
+ }
+
+ @Override
+ public void tearDownCluster() {
+ for (DocumentMK kernel : kernels) {
+ kernel.dispose();
+ }
+ if (dropDBAfterTest) {
+ try {
+ MongoConnection mongo =
+ new MongoConnection(uri);
+ mongo.getDB().dropDatabase();
+ mongo.close();
+ if(blobStoreFixture != null){
+ blobStoreFixture.tearDown();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ private void setupBlobStore(DocumentMK.Builder mkBuilder, StatisticsProvider statsProvider) {
+ initializeBlobStoreFixture(statsProvider);
+ if (blobStoreFixture != null) {
+ mkBuilder.setBlobStore(blobStoreFixture.setUp());
+ }
+ }
+
+ private void initializeBlobStoreFixture(StatisticsProvider statsProvider) {
+ if (blobStoreFixture != null){
+ return;
+ }
+
+ if (useDataStore) {
+ blobStoreFixture =
+ BlobStoreFixture.create(base, true, dsCacheInMB, statsProvider);
+ }
+ }
+
+ }
+
public static class SegmentFixture extends OakFixture {
private FileStore[] stores;
private BlobStoreFixture[] blobStoreFixtures = new BlobStoreFixture[0];
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java?rev=1767237&r1=1767236&r2=1767237&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreHelper.java Mon Oct 31 10:12:36 2016
@@ -28,6 +28,7 @@ import javax.annotation.Nullable;
import com.google.common.base.Function;
import com.google.common.base.Stopwatch;
+import com.google.common.cache.Cache;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.primitives.Longs;
@@ -54,6 +55,10 @@ import static org.apache.jackrabbit.oak.
*/
public class DocumentNodeStoreHelper {
+ public static Cache<PathRev, DocumentNodeState> getNodesCache(DocumentNodeStore dns) {
+ return dns.getNodeCache();
+ }
+
public static void garbageReport(DocumentNodeStore dns) {
System.out.print("Collecting top 100 nodes with most blob garbage ");
Stopwatch sw = Stopwatch.createStarted();