You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2017/07/20 17:09:14 UTC
hbase git commit: HBASE-17738 BucketCache startup is slow - addendum
(Ram)
Repository: hbase
Updated Branches:
refs/heads/master 01db60d65 -> 0c49185c3
HBASE-17738 BucketCache startup is slow - addendum (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0c49185c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0c49185c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0c49185c
Branch: refs/heads/master
Commit: 0c49185c3e95cc91ba6455a404ca4e89f2c2fc20
Parents: 01db60d
Author: Ramkrishna <ra...@intel.com>
Authored: Thu Jul 20 22:38:13 2017 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Thu Jul 20 22:38:13 2017 +0530
----------------------------------------------------------------------
.../hadoop/hbase/util/ByteBufferArray.java | 17 ++++++---
.../hadoop/hbase/util/TestByteBufferArray.java | 39 ++++++++++++++++++++
2 files changed, 51 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0c49185c/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferArray.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferArray.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferArray.java
index 60f8c79..068afe2 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferArray.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferArray.java
@@ -44,14 +44,15 @@ import com.google.common.annotations.VisibleForTesting;
* reading/writing data from this large buffer with a position and offset
*/
@InterfaceAudience.Private
-public final class ByteBufferArray {
+public class ByteBufferArray {
private static final Log LOG = LogFactory.getLog(ByteBufferArray.class);
public static final int DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
@VisibleForTesting
ByteBuffer buffers[];
private int bufferSize;
- private int bufferCount;
+ @VisibleForTesting
+ int bufferCount;
/**
* We allocate a number of byte buffers as the capacity. In order not to out
@@ -75,12 +76,13 @@ public final class ByteBufferArray {
createBuffers(directByteBuffer, allocator);
}
- private void createBuffers(boolean directByteBuffer, ByteBufferAllocator allocator)
+ @VisibleForTesting
+ void createBuffers(boolean directByteBuffer, ByteBufferAllocator allocator)
throws IOException {
- int threadCount = Runtime.getRuntime().availableProcessors();
+ int threadCount = getThreadCount();
ExecutorService service = new ThreadPoolExecutor(threadCount, threadCount, 0L,
TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>());
- int perThreadCount = Math.round((float) (bufferCount) / threadCount);
+ int perThreadCount = (int)Math.floor((double) (bufferCount) / threadCount);
int lastThreadCount = bufferCount - (perThreadCount * (threadCount - 1));
Future<ByteBuffer[]>[] futures = new Future[threadCount];
try {
@@ -109,6 +111,11 @@ public final class ByteBufferArray {
this.buffers[bufferCount] = ByteBuffer.allocate(0);
}
+ @VisibleForTesting
+ int getThreadCount() {
+ return Runtime.getRuntime().availableProcessors();
+ }
+
/**
* A callable that creates buffers of the specified length either onheap/offheap using the
* {@link ByteBufferAllocator}
http://git-wip-us.apache.org/repos/asf/hbase/blob/0c49185c/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferArray.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferArray.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferArray.java
index c71b86c..7077643 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferArray.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferArray.java
@@ -79,4 +79,43 @@ public class TestByteBufferArray {
}
}
}
+
+ @Test
+ public void testByteBufferCreation1() throws Exception {
+ ByteBufferAllocator allocator = new ByteBufferAllocator() {
+ @Override
+ public ByteBuffer allocate(long size, boolean directByteBuffer) throws IOException {
+ if (directByteBuffer) {
+ return ByteBuffer.allocateDirect((int) size);
+ } else {
+ return ByteBuffer.allocate((int) size);
+ }
+ }
+ };
+ ByteBufferArray array = new DummyByteBufferArray(7 * 1024 * 1024, false, allocator);
+ // overwrite
+ array.bufferCount = 25;
+ array.buffers = new ByteBuffer[array.bufferCount + 1];
+ array.createBuffers(true, allocator);
+ for (int i = 0; i < array.buffers.length; i++) {
+ if (i == array.buffers.length - 1) {
+ assertEquals(array.buffers[i].capacity(), 0);
+ } else {
+ assertEquals(array.buffers[i].capacity(), 458752);
+ }
+ }
+ }
+
+ private static class DummyByteBufferArray extends ByteBufferArray {
+
+ public DummyByteBufferArray(long capacity, boolean directByteBuffer,
+ ByteBufferAllocator allocator) throws IOException {
+ super(capacity, directByteBuffer, allocator);
+ }
+
+ @Override
+ int getThreadCount() {
+ return 16;
+ }
+ }
}