You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2016/09/07 16:09:53 UTC
svn commit: r1759635 - in
/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby:
BulkTest.java StandbyTest.java StandbyTestIT.java benchmark/
benchmark/BenchmarkBase.java benchmark/BulkTransferBenchmark.java
Author: frm
Date: Wed Sep 7 16:09:53 2016
New Revision: 1759635
URL: http://svn.apache.org/viewvc?rev=1759635&view=rev
Log:
OAK-4745 - Turn BulkTest in a benchmark utility class
Contribution by Andrei Dulceanu.
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java (with props)
Removed:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/BulkTest.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java?rev=1759635&r1=1759634&r2=1759635&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTest.java Wed Sep 7 16:09:53 2016
@@ -28,7 +28,6 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Random;
-import com.google.common.io.ByteStreams;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.PropertyState;
@@ -43,9 +42,10 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
+import com.google.common.io.ByteStreams;
+
public class StandbyTest extends TestBase {
@Before
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java?rev=1759635&r1=1759634&r2=1759635&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbyTestIT.java Wed Sep 7 16:09:53 2016
@@ -43,7 +43,6 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
public class StandbyTestIT extends TestBase {
Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java?rev=1759635&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java Wed Sep 7 16:09:53 2016
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment.standby.benchmark;
+
+import static java.io.File.createTempFile;
+import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.standby.client.StandbyClient;
+import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+
+public class BenchmarkBase {
+ static final int port = Integer.getInteger("standby.server.port", 52800);
+ static final String LOCALHOST = "127.0.0.1";
+
+ static final int timeout = Integer.getInteger("standby.test.timeout", 500);
+
+ File directoryS;
+ FileStore storeS;
+ ScheduledExecutorService executorS;
+
+ File directoryC;
+ FileStore storeC;
+ ScheduledExecutorService executorC;
+
+ public void setUpServerAndClient() throws Exception {
+ directoryS = createTmpTargetDir(getClass().getSimpleName() + "-Server");
+ executorS = Executors.newSingleThreadScheduledExecutor();
+ storeS = setupPrimary(directoryS, executorS);
+
+ // client
+ directoryC = createTmpTargetDir(getClass().getSimpleName() + "-Client");
+ executorC = Executors.newSingleThreadScheduledExecutor();
+ storeC = setupSecondary(directoryC, executorC);
+ }
+
+ public void closeServerAndClient() {
+ storeS.close();
+ storeC.close();
+
+ try {
+ FileUtils.deleteDirectory(directoryS);
+ FileUtils.deleteDirectory(directoryC);
+ } catch (IOException e) {
+ // ignore
+ } finally {
+ if (executorS != null) {
+ new ExecutorCloser(executorS).close();
+ }
+
+ if (executorC != null) {
+ new ExecutorCloser(executorC).close();
+ }
+ }
+ }
+
+ private static FileStore newFileStore(File directory, ScheduledExecutorService executor) throws Exception {
+ return fileStoreBuilder(directory).withMaxFileSize(1).withMemoryMapping(false).withNodeDeduplicationCacheSize(0)
+ .withSegmentCacheSize(0).withStringCacheSize(0).withTemplateCacheSize(0)
+ .withStatisticsProvider(new DefaultStatisticsProvider(executor)).build();
+ }
+
+ protected FileStore setupPrimary(File directory, ScheduledExecutorService executor) throws Exception {
+ return newFileStore(directory, executor);
+ }
+
+ protected FileStore setupSecondary(File directory, ScheduledExecutorService executor) throws Exception {
+ return newFileStore(directory, executor);
+ }
+
+ public StandbyClient newStandbyClient(FileStore store) throws Exception {
+ return newStandbyClient(store, port, false);
+ }
+
+ public StandbyClient newStandbyClient(FileStore store, int port) throws Exception {
+ return newStandbyClient(store, port, false);
+ }
+
+ public StandbyClient newStandbyClient(FileStore store, int port, boolean secure) throws Exception {
+ return new StandbyClient(LOCALHOST, port, store, secure, timeout, false);
+ }
+
+ private static File createTmpTargetDir(String name) throws IOException {
+ File f = createTempFile(name, "dir", new File("target"));
+ f.delete();
+ f.mkdir();
+ return f;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BenchmarkBase.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java?rev=1759635&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java Wed Sep 7 16:09:53 2016
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.standby.benchmark;
+
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Method;
+import java.util.Set;
+
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.standby.client.StandbyClient;
+import org.apache.jackrabbit.oak.segment.standby.jmx.StandbyStatusMBean;
+import org.apache.jackrabbit.oak.segment.standby.server.StandbyServer;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+public class BulkTransferBenchmark extends BenchmarkBase {
+
+ public void setUp() throws Exception {
+ setUpServerAndClient();
+ }
+
+ public void after() {
+ closeServerAndClient();
+ }
+
+ public void test100Nodes() throws Exception {
+ test(100, 1, 1, 3000, 3100);
+ }
+
+ public void test1000Nodes() throws Exception {
+ test(1000, 1, 1, 53000, 55000);
+ }
+
+ public void test10000Nodes() throws Exception {
+ test(10000, 1, 1, 245000, 246000);
+ }
+
+ public void test100000Nodes() throws Exception {
+ test(100000, 9, 9, 2210000, 2220000);
+ }
+
+ public void test1MillionNodes() throws Exception {
+ test(1000000, 87, 87, 22700000, 22800000);
+ }
+
+ public void test1MillionNodesUsingSSL() throws Exception {
+ test(1000000, 87, 87, 22700000, 22800000, true);
+ }
+
+ public void test10MillionNodes() throws Exception {
+ test(10000000, 856, 856, 223000000, 224000000);
+ }
+
+ private void test(int number, int minExpectedSegments, int maxExpectedSegments, long minExpectedBytes, long maxExpectedBytes) throws Exception {
+ test(number, minExpectedSegments, maxExpectedSegments, minExpectedBytes, maxExpectedBytes, false);
+ }
+
+ private void test(int number, int minExpectedSegments, int maxExpectedSegments, long minExpectedBytes, long maxExpectedBytes,
+ boolean useSSL) throws Exception {
+ NodeStore store = SegmentNodeStoreBuilders.builder(storeS).build();
+ NodeBuilder rootbuilder = store.getRoot().builder();
+ NodeBuilder b = rootbuilder.child("store");
+ for (int j=0; j<=number / 1000; j++) {
+ NodeBuilder builder = b.child("Folder#" + j);
+ for (int i = 0; i <(number < 1000 ? number : 1000); i++) {
+ builder.child("Test#" + i).setProperty("ts", System.currentTimeMillis());
+ }
+ }
+ store.merge(rootbuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ storeS.flush();
+
+ final StandbyServer server = new StandbyServer(port, storeS, useSSL);
+ server.start();
+
+ System.setProperty(StandbyClient.CLIENT_ID_PROPERTY_NAME, "Bar");
+ StandbyClient cl = newStandbyClient(storeC, port, useSSL);
+
+ final MBeanServer jmxServer = ManagementFactory.getPlatformMBeanServer();
+ ObjectName status = new ObjectName(StandbyStatusMBean.JMX_NAME + ",id=*");
+ ObjectName clientStatus = new ObjectName(cl.getMBeanName());
+ ObjectName serverStatus = new ObjectName(server.getMBeanName());
+
+ long start = System.currentTimeMillis();
+ cl.run();
+
+ try {
+ Set<ObjectName> instances = jmxServer.queryNames(status, null);
+ ObjectName connectionStatus = null;
+ for (ObjectName s : instances) {
+ if (!s.equals(clientStatus) && !s.equals(serverStatus)) connectionStatus = s;
+ }
+ assert(connectionStatus != null);
+
+ long segments = ((Long)jmxServer.getAttribute(connectionStatus, "TransferredSegments")).longValue();
+ long bytes = ((Long)jmxServer.getAttribute(connectionStatus, "TransferredSegmentBytes")).longValue();
+
+ System.out.println("did transfer " + segments + " segments with " + bytes + " bytes in " + (System.currentTimeMillis() - start) / 1000 + " seconds.");
+ } finally {
+ server.close();
+ cl.close();
+ }
+ }
+
+ public static void main(String[] args) {
+ BulkTransferBenchmark benchmark = new BulkTransferBenchmark();
+
+ String[] methodNames = new String[] {
+ "test100Nodes",
+ "test1000Nodes",
+ "test10000Nodes",
+ "test100000Nodes",
+ "test1MillionNodes",
+ "test1MillionNodesUsingSSL",
+ "test10MillionNodes"
+ };
+
+ for (String methodName : methodNames) {
+ try {
+ Method method = benchmark.getClass().getMethod(methodName);
+
+ benchmark.setUp();
+ method.invoke(benchmark);
+ benchmark.after();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/benchmark/BulkTransferBenchmark.java
------------------------------------------------------------------------------
svn:eol-style = native