You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2017/07/10 19:02:37 UTC
[1/3] accumulo git commit: ACCUMULO-4602 Improvements to
AssignmentThreadsIT
Repository: accumulo
Updated Branches:
refs/heads/master 0b7927ba9 -> 2415f22be
ACCUMULO-4602 Improvements to AssignmentThreadsIT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b7d9be09
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b7d9be09
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b7d9be09
Branch: refs/heads/master
Commit: b7d9be0954accdf6ac91cd541f4ed9e60e49f04a
Parents: 9e7e358
Author: Mike Miller <mm...@apache.org>
Authored: Mon Jul 10 14:05:54 2017 -0400
Committer: Mike Miller <mm...@apache.org>
Committed: Mon Jul 10 14:05:54 2017 -0400
----------------------------------------------------------------------
.../accumulo/test/AssignmentThreadsIT.java | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b7d9be09/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java b/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java
index 133cabd..3778de6 100644
--- a/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/AssignmentThreadsIT.java
@@ -71,23 +71,27 @@ public class AssignmentThreadsIT extends ConfigurableMacIT {
c.tableOperations().addSplits(tableName, splits);
log.info("Taking table offline");
c.tableOperations().offline(tableName, true);
+
// time how long it takes to load
log.info("Bringing the table online");
- long now = System.currentTimeMillis();
+ long now = System.nanoTime();
c.tableOperations().online(tableName, true);
- long diff = System.currentTimeMillis() - now;
- log.info("Loaded " + splits.size() + " tablets in " + diff + " ms");
+ long diff = System.nanoTime() - now;
+
+ log.info("Loaded " + splits.size() + " tablets in " + diff + " ns");
c.instanceOperations().setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT.getKey(), "20");
- now = System.currentTimeMillis();
log.info("Taking table offline, again");
c.tableOperations().offline(tableName, true);
// wait >10 seconds for thread pool to update
- UtilWaitThread.sleep(Math.max(0, now + 11 * 1000 - System.currentTimeMillis()));
- now = System.currentTimeMillis();
+ log.info("sleep for {} ms so thread pool can update", 60_000);
+ UtilWaitThread.sleep(60_000);
+
log.info("Bringing table back online");
+ now = System.nanoTime();
c.tableOperations().online(tableName, true);
- long diff2 = System.currentTimeMillis() - now;
- log.debug("Loaded " + splits.size() + " tablets in " + diff2 + " ms");
+ long diff2 = System.nanoTime() - now;
+
+ log.debug("Loaded " + splits.size() + " tablets in " + diff2 + " ns");
assertTrue(diff2 < diff);
}
[2/3] accumulo git commit: Merge branch '1.7' into 1.8
Posted by mm...@apache.org.
Merge branch '1.7' into 1.8
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cdafd020
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cdafd020
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cdafd020
Branch: refs/heads/master
Commit: cdafd0203469aa51ba120b1ac0ed34cc579eaa18
Parents: fab5589 b7d9be0
Author: Mike Miller <mm...@apache.org>
Authored: Mon Jul 10 14:28:42 2017 -0400
Committer: Mike Miller <mm...@apache.org>
Committed: Mon Jul 10 14:28:42 2017 -0400
----------------------------------------------------------------------
.../accumulo/test/AssignmentThreadsIT.java | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cdafd020/test/src/main/java/org/apache/accumulo/test/AssignmentThreadsIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/AssignmentThreadsIT.java
index bbc1d47,0000000..fafe349
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/AssignmentThreadsIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/AssignmentThreadsIT.java
@@@ -1,99 -1,0 +1,103 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.categories.MiniClusterOnlyTests;
+import org.apache.accumulo.test.categories.PerformanceTests;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+// ACCUMULO-1177
+@Category({MiniClusterOnlyTests.class, PerformanceTests.class})
+public class AssignmentThreadsIT extends ConfigurableMacBase {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
+ cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "1");
+ }
+
+ // [0-9a-f]
+ private final static byte[] HEXCHARS = {0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66};
+ private final static Random random = new Random();
+
+ public static byte[] randomHex(int n) {
+ byte[] binary = new byte[n];
+ byte[] hex = new byte[n * 2];
+ random.nextBytes(binary);
+ int count = 0;
+ for (byte x : binary) {
+ hex[count++] = HEXCHARS[(x >> 4) & 0xf];
+ hex[count++] = HEXCHARS[x & 0xf];
+ }
+ return hex;
+ }
+
+ @Test(timeout = 5 * 60 * 1000)
+ public void testConcurrentAssignmentPerformance() throws Exception {
+ // make a table with a lot of splits
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ log.info("Creating table");
+ c.tableOperations().create(tableName);
+ SortedSet<Text> splits = new TreeSet<>();
+ for (int i = 0; i < 1000; i++) {
+ splits.add(new Text(randomHex(8)));
+ }
+ log.info("Adding splits");
+ c.tableOperations().addSplits(tableName, splits);
+ log.info("Taking table offline");
+ c.tableOperations().offline(tableName, true);
++
+ // time how long it takes to load
+ log.info("Bringing the table online");
- long now = System.currentTimeMillis();
++ long now = System.nanoTime();
+ c.tableOperations().online(tableName, true);
- long diff = System.currentTimeMillis() - now;
- log.info("Loaded " + splits.size() + " tablets in " + diff + " ms");
++ long diff = System.nanoTime() - now;
++
++ log.info("Loaded " + splits.size() + " tablets in " + diff + " ns");
+ c.instanceOperations().setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT.getKey(), "20");
- now = System.currentTimeMillis();
+ log.info("Taking table offline, again");
+ c.tableOperations().offline(tableName, true);
+ // wait >10 seconds for thread pool to update
- sleepUninterruptibly(Math.max(0, now + 11 * 1000 - System.currentTimeMillis()), TimeUnit.MILLISECONDS);
- now = System.currentTimeMillis();
++ log.info("sleep for {} ms so thread pool can update", 60_000);
++ sleepUninterruptibly(60_000, TimeUnit.MILLISECONDS);
++
+ log.info("Bringing table back online");
++ now = System.nanoTime();
+ c.tableOperations().online(tableName, true);
- long diff2 = System.currentTimeMillis() - now;
- log.debug("Loaded " + splits.size() + " tablets in " + diff2 + " ms");
++ long diff2 = System.nanoTime() - now;
++
++ log.debug("Loaded " + splits.size() + " tablets in " + diff2 + " ns");
+ assertTrue(diff2 < diff);
+ }
+
+}
[3/3] accumulo git commit: Merge branch '1.8'
Posted by mm...@apache.org.
Merge branch '1.8'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2415f22b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2415f22b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2415f22b
Branch: refs/heads/master
Commit: 2415f22be51b788a6b98518c8e3622dc8742dce0
Parents: 0b7927b cdafd02
Author: Mike Miller <mm...@apache.org>
Authored: Mon Jul 10 15:01:52 2017 -0400
Committer: Mike Miller <mm...@apache.org>
Committed: Mon Jul 10 15:01:52 2017 -0400
----------------------------------------------------------------------
.../accumulo/test/AssignmentThreadsIT.java | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------