You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by ch...@apache.org on 2008/07/17 22:41:42 UTC
svn commit: r677715 - in
/activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer:
BTreeFilerBenchmark.java FilerBenchmark.java HashFilerBenchmark.java
Author: chirino
Date: Thu Jul 17 13:41:41 2008
New Revision: 677715
URL: http://svn.apache.org/viewvc?rev=677715&view=rev
Log:
Adding a little benchmark tool that can be used to measure the performance of the Filer implementations.
Added:
activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java
activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java (with props)
activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java
Added: activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java
URL: http://svn.apache.org/viewvc/activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java?rev=677715&view=auto
==============================================================================
--- activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java (added)
+++ activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/BTreeFilerBenchmark.java Thu Jul 17 13:41:41 2008
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.xindice.core.filer;
+
+import java.io.File;
+
+public class BTreeFilerBenchmark extends FilerBenchmark {
+
+ @Override
+ protected Filer createFiler(File root, String name) throws Exception {
+ BTreeFiler rc = new BTreeFiler();
+ rc.setLocation(ROOT_DIR, name);
+ rc.setPageCount(10000);
+ rc.setPageSize(4096);
+ return rc;
+ }
+
+}
Added: activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java
URL: http://svn.apache.org/viewvc/activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java?rev=677715&view=auto
==============================================================================
--- activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java (added)
+++ activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java Thu Jul 17 13:41:41 2008
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.xindice.core.filer;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import junit.framework.TestCase;
+import org.apache.xindice.core.data.Key;
+import org.apache.xindice.core.data.Record;
+import org.apache.xindice.core.data.Value;
+
+/**
+ * @author chirino
+ */
+public abstract class FilerBenchmark extends TestCase {
+
+ // Slower machines might need to make this bigger.
+ private static final long SAMPLE_DURATION = Integer.parseInt(System.getProperty("SAMPLES_DURATION", "" + 1000 * 5));
+ // How many times do we sample?
+ private static final long SAMPLES = Integer.parseInt(System.getProperty("SAMPLES", "" + 60 * 1000 / SAMPLE_DURATION));
+
+ private static final int FILER_COUNT = Integer.parseInt(System.getProperty("FILER_COUNT", "" + 1));
+ private static final int FILER_PRE_LOAD_COUNT = Integer.parseInt(System.getProperty("FILER_PRE_LOAD_COUNT", "" + 10000 / FILER_COUNT));
+
+ protected File ROOT_DIR;
+ protected final HashMap<String, Filer> indexes = new HashMap<String, Filer>();
+
+ public void setUp() throws Exception {
+ ROOT_DIR = new File("target/test-data/" + getClass().getName());
+ ROOT_DIR.mkdirs();
+ }
+
+ protected void tearDown() throws Exception {
+ for (Filer f : indexes.values()) {
+ try {
+ f.close();
+ } catch (Throwable ignore) {
+ }
+ }
+ }
+
+ abstract protected Filer createFiler(File root, String name) throws Exception;
+
+ synchronized private Filer openFiler(String name) throws Exception, FilerException {
+ Filer index = indexes.get(name);
+ if (index == null) {
+ index = createFiler(ROOT_DIR, name);
+ if (!index.exists()) {
+ index.create();
+ }
+ index.open();
+ indexes.put(name, index);
+ }
+ return index;
+ }
+
+ class Producer extends Thread {
+ private final String name;
+ AtomicBoolean shutdown = new AtomicBoolean();
+
+ public Producer(String name) {
+ super("Producer: " + name);
+ this.name = name;
+ }
+
+ public void shutdown() {
+ shutdown.set(true);
+ }
+
+ @Override
+ public void run() {
+ try {
+ Filer filer = openFiler(name);
+ long counter = 0;
+ Value value = new Value(new byte[] {
+ 1, 2, 3, 4, 5, 6, 7, 8
+ });
+ while (!shutdown.get()) {
+ long c = counter;
+ Key key = new Key("a-long-message-id-like-key-" + c);
+ filer.writeRecord(key, value);
+ onProduced(counter++);
+ }
+
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void onProduced(long counter) {
+ }
+ }
+
+ class Consumer extends Thread {
+ private final String name;
+ AtomicBoolean shutdown = new AtomicBoolean();
+
+ public Consumer(String name) {
+ super("Consumer: " + name);
+ this.name = name;
+ }
+
+ public void shutdown() {
+ shutdown.set(true);
+ }
+
+ @Override
+ public void run() {
+ try {
+ Filer filer = openFiler(name);
+ long counter = 0;
+ while (!shutdown.get()) {
+ long c = counter;
+ Key key = new Key("a-long-message-id-like-key-" + c);
+ Record record;
+ record = filer.readRecord(key);
+ if (record != null) {
+ filer.deleteRecord(key);
+ onConsumed(counter++);
+ } else {
+ Thread.sleep(0);
+ }
+ }
+ } catch (Throwable e) {
+ e.printStackTrace();
+ }
+ }
+
+ public void onConsumed(long counter) {
+ }
+ }
+
+ public void testLoad() throws Exception {
+
+ final Producer producers[] = new Producer[FILER_COUNT];
+ final Consumer consumers[] = new Consumer[FILER_COUNT];
+ final CountDownLatch preloadCountDown = new CountDownLatch(FILER_COUNT);
+ final AtomicLong producedRecords = new AtomicLong();
+ final AtomicLong consumedRecords = new AtomicLong();
+
+ System.out.println("Starting: " + FILER_COUNT + " producers");
+ for (int i = 0; i < FILER_COUNT; i++) {
+ producers[i] = new Producer("test-" + i) {
+ private boolean prelaodDone;
+
+ public void onProduced(long counter) {
+ if (!prelaodDone && counter >= FILER_PRE_LOAD_COUNT) {
+ prelaodDone = true;
+ preloadCountDown.countDown();
+ }
+ producedRecords.incrementAndGet();
+ }
+ };
+ producers[i].start();
+ }
+
+ long start = System.currentTimeMillis();
+ System.out.println("Waiting for each producer create " + FILER_PRE_LOAD_COUNT + " records before starting the consumers.");
+ preloadCountDown.await();
+ long end = System.currentTimeMillis();
+ System.out.println("Preloaded " + FILER_PRE_LOAD_COUNT * FILER_COUNT + " records at " + (FILER_PRE_LOAD_COUNT * FILER_COUNT * 1000f / (end - start)) + " records/sec");
+
+ System.out.println("Starting: " + FILER_COUNT + " consumers");
+ for (int i = 0; i < FILER_COUNT; i++) {
+ consumers[i] = new Consumer("test-" + i) {
+ public void onConsumed(long counter) {
+ consumedRecords.incrementAndGet();
+ }
+ };
+ consumers[i].start();
+ }
+
+ long sample_start = System.currentTimeMillis();
+ System.out.println("Taking " + SAMPLES + " performance samples every " + SAMPLE_DURATION + " ms");
+ System.out.println("time (s), produced, produce rate (r/s), consumed, consume rate (r/s), used memory (k)");
+ producedRecords.set(0);
+ consumedRecords.set(0);
+ for (int i = 0; i < SAMPLES; i++) {
+ start = System.currentTimeMillis();
+ Thread.sleep(SAMPLE_DURATION);
+ end = System.currentTimeMillis();
+ long p = producedRecords.getAndSet(0);
+ long c = consumedRecords.getAndSet(0);
+
+ long usedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
+
+ System.out.println(((end - sample_start) / 1000f) + ", " + p + ", " + (p * 1000f / (end - start)) + ", " + c + ", " + (c * 1000f / (end - start)) + ", " + (usedMemory / (1024)));
+ }
+ System.out.println("Samples done... Shutting down the producers and consumers...");
+ for (int i = 0; i < FILER_COUNT; i++) {
+ producers[i].shutdown();
+ consumers[i].shutdown();
+ }
+ for (int i = 0; i < FILER_COUNT; i++) {
+ producers[i].join(1000 * 5);
+ consumers[i].join(1000 * 5);
+ }
+ System.out.println("Shutdown.");
+ }
+
+}
Propchange: activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/FilerBenchmark.java
------------------------------------------------------------------------------
svn:executable = *
Added: activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java
URL: http://svn.apache.org/viewvc/activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java?rev=677715&view=auto
==============================================================================
--- activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java (added)
+++ activemq/sandbox/xindice-stripped/src/test/java/org/apache/xindice/core/filer/HashFilerBenchmark.java Thu Jul 17 13:41:41 2008
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.xindice.core.filer;
+
+import java.io.File;
+
+public class HashFilerBenchmark extends FilerBenchmark {
+
+ @Override
+ protected Filer createFiler(File root, String name) throws Exception {
+ HashFiler rc = new HashFiler();
+ rc.setLocation(ROOT_DIR, name);
+ rc.setPageCount(10000);
+ rc.setPageSize(4096);
+ return rc;
+ }
+
+}