You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2015/05/22 04:45:45 UTC
[1/3] accumulo git commit: ACCUMULO-3845 Run each write test multiple
times per durability.
Repository: accumulo
Updated Branches:
refs/heads/1.7 747da5a9a -> 2eeaebb69
refs/heads/master 350b491d9 -> 19bacd292
ACCUMULO-3845 Run each write test multiple times per durability.
This test can be a little sensitive to performance
outliers. If we run each ingest cycle at each durability
multiple times and take the median, we should have a
more reliable test.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2eeaebb6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2eeaebb6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2eeaebb6
Branch: refs/heads/1.7
Commit: 2eeaebb692cbb9a649a7e20da34dfd3f3cf891f3
Parents: 747da5a
Author: Josh Elser <el...@apache.org>
Authored: Thu May 21 22:43:23 2015 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 21 22:45:22 2015 -0400
----------------------------------------------------------------------
.../accumulo/test/functional/DurabilityIT.java | 44 +++++++++++++-------
1 file changed, 29 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2eeaebb6/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java b/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
index f2ba949..aa280dc 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -36,10 +37,13 @@ import org.apache.accumulo.minicluster.impl.ProcessReference;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterators;
public class DurabilityIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(DurabilityIT.class);
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
@@ -82,17 +86,21 @@ public class DurabilityIT extends ConfigurableMacIT {
TableOperations tableOps = getConnector().tableOperations();
String tableNames[] = init();
// write some gunk, delete the table to keep that table from messing with the performance numbers of successive calls
+ // sync
long t0 = writeSome(tableNames[0], N);
tableOps.delete(tableNames[0]);
+ // flush
long t1 = writeSome(tableNames[1], N);
tableOps.delete(tableNames[1]);
+ // log
long t2 = writeSome(tableNames[2], N);
tableOps.delete(tableNames[2]);
+ // none
long t3 = writeSome(tableNames[3], N);
tableOps.delete(tableNames[3]);
System.out.println(String.format("sync %d flush %d log %d none %d", t0, t1, t2, t3));
- assertTrue("flush-only should be faster than sync", t0 > t1);
- assertTrue("sync should be faster than log", t1 > t2);
+ assertTrue("flush should be faster than sync", t0 > t1);
+ assertTrue("log should be faster than flush", t1 > t2);
assertTrue("no durability should be faster than log", t2 > t3);
}
@@ -185,21 +193,27 @@ public class DurabilityIT extends ConfigurableMacIT {
}
private long writeSome(String table, long count) throws Exception {
- long now = System.currentTimeMillis();
- Connector c = getConnector();
- BatchWriter bw = c.createBatchWriter(table, null);
- for (int i = 1; i < count + 1; i++) {
- Mutation m = new Mutation("" + i);
- m.put("", "", "");
- bw.addMutation(m);
- if (i % (Math.max(1, count / 100)) == 0) {
- bw.flush();
+ int iterations = 5;
+ long[] attempts = new long[iterations];
+ for (int attempt = 0; attempt < iterations; attempt++) {
+ long now = System.currentTimeMillis();
+ Connector c = getConnector();
+ BatchWriter bw = c.createBatchWriter(table, null);
+ for (int i = 1; i < count + 1; i++) {
+ Mutation m = new Mutation("" + i);
+ m.put("", "", "");
+ bw.addMutation(m);
+ if (i % (Math.max(1, count / 100)) == 0) {
+ bw.flush();
+ }
}
+ bw.close();
+ attempts[attempt] = System.currentTimeMillis() - now;
}
- bw.close();
- long result = System.currentTimeMillis() - now;
- // c.tableOperations().flush(table, null, null, true);
- return result;
+ Arrays.sort(attempts);
+ log.info("Attempt durations: {}", Arrays.toString(attempts));
+ // Return the median duration
+ return attempts[2];
}
}
[3/3] accumulo git commit: Merge branch '1.7'
Posted by el...@apache.org.
Merge branch '1.7'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/19bacd29
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/19bacd29
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/19bacd29
Branch: refs/heads/master
Commit: 19bacd2922e4cf752c2b1b0fc4dfdfd891b29dec
Parents: 350b491 2eeaebb
Author: Josh Elser <el...@apache.org>
Authored: Thu May 21 22:45:33 2015 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 21 22:45:33 2015 -0400
----------------------------------------------------------------------
.../accumulo/test/functional/DurabilityIT.java | 44 +++++++++++++-------
1 file changed, 29 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
[2/3] accumulo git commit: ACCUMULO-3845 Run each write test multiple
times per durability.
Posted by el...@apache.org.
ACCUMULO-3845 Run each write test multiple times per durability.
This test can be a little sensitive to performance
outliers. If we run each ingest cycle at each durability
multiple times and take the median, we should have a
more reliable test.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2eeaebb6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2eeaebb6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2eeaebb6
Branch: refs/heads/master
Commit: 2eeaebb692cbb9a649a7e20da34dfd3f3cf891f3
Parents: 747da5a
Author: Josh Elser <el...@apache.org>
Authored: Thu May 21 22:43:23 2015 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 21 22:45:22 2015 -0400
----------------------------------------------------------------------
.../accumulo/test/functional/DurabilityIT.java | 44 +++++++++++++-------
1 file changed, 29 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2eeaebb6/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java b/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
index f2ba949..aa280dc 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -36,10 +37,13 @@ import org.apache.accumulo.minicluster.impl.ProcessReference;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterators;
public class DurabilityIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(DurabilityIT.class);
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
@@ -82,17 +86,21 @@ public class DurabilityIT extends ConfigurableMacIT {
TableOperations tableOps = getConnector().tableOperations();
String tableNames[] = init();
// write some gunk, delete the table to keep that table from messing with the performance numbers of successive calls
+ // sync
long t0 = writeSome(tableNames[0], N);
tableOps.delete(tableNames[0]);
+ // flush
long t1 = writeSome(tableNames[1], N);
tableOps.delete(tableNames[1]);
+ // log
long t2 = writeSome(tableNames[2], N);
tableOps.delete(tableNames[2]);
+ // none
long t3 = writeSome(tableNames[3], N);
tableOps.delete(tableNames[3]);
System.out.println(String.format("sync %d flush %d log %d none %d", t0, t1, t2, t3));
- assertTrue("flush-only should be faster than sync", t0 > t1);
- assertTrue("sync should be faster than log", t1 > t2);
+ assertTrue("flush should be faster than sync", t0 > t1);
+ assertTrue("log should be faster than flush", t1 > t2);
assertTrue("no durability should be faster than log", t2 > t3);
}
@@ -185,21 +193,27 @@ public class DurabilityIT extends ConfigurableMacIT {
}
private long writeSome(String table, long count) throws Exception {
- long now = System.currentTimeMillis();
- Connector c = getConnector();
- BatchWriter bw = c.createBatchWriter(table, null);
- for (int i = 1; i < count + 1; i++) {
- Mutation m = new Mutation("" + i);
- m.put("", "", "");
- bw.addMutation(m);
- if (i % (Math.max(1, count / 100)) == 0) {
- bw.flush();
+ int iterations = 5;
+ long[] attempts = new long[iterations];
+ for (int attempt = 0; attempt < iterations; attempt++) {
+ long now = System.currentTimeMillis();
+ Connector c = getConnector();
+ BatchWriter bw = c.createBatchWriter(table, null);
+ for (int i = 1; i < count + 1; i++) {
+ Mutation m = new Mutation("" + i);
+ m.put("", "", "");
+ bw.addMutation(m);
+ if (i % (Math.max(1, count / 100)) == 0) {
+ bw.flush();
+ }
}
+ bw.close();
+ attempts[attempt] = System.currentTimeMillis() - now;
}
- bw.close();
- long result = System.currentTimeMillis() - now;
- // c.tableOperations().flush(table, null, null, true);
- return result;
+ Arrays.sort(attempts);
+ log.info("Attempt durations: {}", Arrays.toString(attempts));
+ // Return the median duration
+ return attempts[2];
}
}