You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/10/09 05:18:52 UTC
[1/3] git commit: ACCUMULO-3213 Instead of trying to start 100
threads at once, do 16 at a time
Repository: accumulo
Updated Branches:
refs/heads/1.6 9b3bc30fe -> 76de76075
refs/heads/master e30594a65 -> aa0cd23be
ACCUMULO-3213 Instead of trying to start 100 threads at once, do 16 at a time
Previously would have been about 50 add splits and 50 delete tables at a time,
this brings it back to 8 and 8, but leaves the same total number of requests
about equal.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76de7607
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76de7607
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76de7607
Branch: refs/heads/1.6
Commit: 76de76075e73c6942069e962c44d7125d58fcae9
Parents: 9b3bc30
Author: Josh Elser <el...@apache.org>
Authored: Wed Oct 8 23:13:01 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Oct 8 23:13:01 2014 -0400
----------------------------------------------------------------------
.../functional/DeleteTableDuringSplitIT.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76de7607/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java b/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
index 707fc19..a0bff64 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertFalse;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -28,11 +29,12 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.fate.util.UtilWaitThread;
import org.apache.hadoop.io.Text;
+import org.junit.Assert;
import org.junit.Test;
// ACCUMULO-2361
public class DeleteTableDuringSplitIT extends SimpleMacIT {
-
+
@Override
protected int defaultTimeoutSeconds() {
return 15 * 60;
@@ -40,7 +42,9 @@ public class DeleteTableDuringSplitIT extends SimpleMacIT {
@Test
public void test() throws Exception {
- String[] tableNames = getUniqueNames(100);
+ // 96 invocations, 8 at a time
+ int batches = 12, batchSize = 8;
+ String[] tableNames = getUniqueNames(batches * batchSize);
// make a bunch of tables
for (String tableName : tableNames) {
getConnector().tableOperations().create(tableName);
@@ -52,7 +56,7 @@ public class DeleteTableDuringSplitIT extends SimpleMacIT {
List<Future<?>> results = new ArrayList<Future<?>>();
List<Runnable> tasks = new ArrayList<Runnable>();
- SimpleThreadPool es = new SimpleThreadPool(tableNames.length, "concurrent-api-requests");
+ SimpleThreadPool es = new SimpleThreadPool(batchSize * 2, "concurrent-api-requests");
for (String tableName : tableNames) {
final String finalName = tableName;
tasks.add(new Runnable() {
@@ -78,11 +82,23 @@ public class DeleteTableDuringSplitIT extends SimpleMacIT {
}
});
}
- for (Runnable r : tasks)
- results.add(es.submit(r));
- for (Future<?> f : results) {
- f.get();
+ Iterator<Runnable> itr = tasks.iterator();
+ for (int batch = 0; batch < batches; batch++) {
+ for (int i = 0; i < batchSize; i++) {
+ Future<?> f = es.submit(itr.next());
+ results.add(f);
+ f = es.submit(itr.next());
+ results.add(f);
+ }
+ for (Future<?> f : results) {
+ f.get();
+ }
+ results.clear();
}
+ // Shut down the ES
+ List<Runnable> queued = es.shutdownNow();
+ Assert.assertTrue("Had more tasks to run", queued.isEmpty());
+ Assert.assertFalse("Had more tasks that needed to be submitted", itr.hasNext());
for (String tableName : tableNames) {
assertFalse(getConnector().tableOperations().exists(tableName));
}
[3/3] git commit: Merge branch '1.6'
Posted by el...@apache.org.
Merge branch '1.6'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/aa0cd23b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/aa0cd23b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/aa0cd23b
Branch: refs/heads/master
Commit: aa0cd23bedeeee137c54bdeca99e3d9ea321dea0
Parents: e30594a 76de760
Author: Josh Elser <el...@apache.org>
Authored: Wed Oct 8 23:16:04 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Oct 8 23:16:04 2014 -0400
----------------------------------------------------------------------
.../functional/DeleteTableDuringSplitIT.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[2/3] git commit: ACCUMULO-3213 Instead of trying to start 100
threads at once, do 16 at a time
Posted by el...@apache.org.
ACCUMULO-3213 Instead of trying to start 100 threads at once, do 16 at a time
Previously would have been about 50 add splits and 50 delete tables at a time,
this brings it back to 8 and 8, but leaves the same total number of requests
about equal.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76de7607
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76de7607
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76de7607
Branch: refs/heads/master
Commit: 76de76075e73c6942069e962c44d7125d58fcae9
Parents: 9b3bc30
Author: Josh Elser <el...@apache.org>
Authored: Wed Oct 8 23:13:01 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Oct 8 23:13:01 2014 -0400
----------------------------------------------------------------------
.../functional/DeleteTableDuringSplitIT.java | 30 +++++++++++++++-----
1 file changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76de7607/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java b/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
index 707fc19..a0bff64 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DeleteTableDuringSplitIT.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertFalse;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -28,11 +29,12 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.fate.util.UtilWaitThread;
import org.apache.hadoop.io.Text;
+import org.junit.Assert;
import org.junit.Test;
// ACCUMULO-2361
public class DeleteTableDuringSplitIT extends SimpleMacIT {
-
+
@Override
protected int defaultTimeoutSeconds() {
return 15 * 60;
@@ -40,7 +42,9 @@ public class DeleteTableDuringSplitIT extends SimpleMacIT {
@Test
public void test() throws Exception {
- String[] tableNames = getUniqueNames(100);
+ // 96 invocations, 8 at a time
+ int batches = 12, batchSize = 8;
+ String[] tableNames = getUniqueNames(batches * batchSize);
// make a bunch of tables
for (String tableName : tableNames) {
getConnector().tableOperations().create(tableName);
@@ -52,7 +56,7 @@ public class DeleteTableDuringSplitIT extends SimpleMacIT {
List<Future<?>> results = new ArrayList<Future<?>>();
List<Runnable> tasks = new ArrayList<Runnable>();
- SimpleThreadPool es = new SimpleThreadPool(tableNames.length, "concurrent-api-requests");
+ SimpleThreadPool es = new SimpleThreadPool(batchSize * 2, "concurrent-api-requests");
for (String tableName : tableNames) {
final String finalName = tableName;
tasks.add(new Runnable() {
@@ -78,11 +82,23 @@ public class DeleteTableDuringSplitIT extends SimpleMacIT {
}
});
}
- for (Runnable r : tasks)
- results.add(es.submit(r));
- for (Future<?> f : results) {
- f.get();
+ Iterator<Runnable> itr = tasks.iterator();
+ for (int batch = 0; batch < batches; batch++) {
+ for (int i = 0; i < batchSize; i++) {
+ Future<?> f = es.submit(itr.next());
+ results.add(f);
+ f = es.submit(itr.next());
+ results.add(f);
+ }
+ for (Future<?> f : results) {
+ f.get();
+ }
+ results.clear();
}
+ // Shut down the ES
+ List<Runnable> queued = es.shutdownNow();
+ Assert.assertTrue("Had more tasks to run", queued.isEmpty());
+ Assert.assertFalse("Had more tasks that needed to be submitted", itr.hasNext());
for (String tableName : tableNames) {
assertFalse(getConnector().tableOperations().exists(tableName));
}