You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/02/22 06:59:14 UTC
[25/50] [abbrv] hbase git commit: HBASE-19506: The CellChunkMap index
chunks are usually small, so in order to prevent memory underutilization,
HBASE-19506 presents small chunks preallocated in a small pool
HBASE-19506: The CellChunkMap index chunks are usually small, so in order to prevent memory underutilization, HBASE-19506 presents small chunks preallocated in a small pool
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/92d04d57
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/92d04d57
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/92d04d57
Branch: refs/heads/HBASE-19064
Commit: 92d04d57516391376d34aaca6307e5906ffa4ae8
Parents: 79d9403
Author: anastas <an...@yahoo-inc.com>
Authored: Wed Feb 21 10:12:25 2018 +0200
Committer: anastas <an...@yahoo-inc.com>
Committed: Wed Feb 21 10:12:25 2018 +0200
----------------------------------------------------------------------
.../regionserver/CellChunkImmutableSegment.java | 76 +++--
.../hadoop/hbase/regionserver/CellChunkMap.java | 17 +-
.../apache/hadoop/hbase/regionserver/Chunk.java | 4 +
.../hadoop/hbase/regionserver/ChunkCreator.java | 324 ++++++++++++++-----
.../regionserver/ImmutableMemStoreLAB.java | 28 +-
.../hadoop/hbase/regionserver/MemStoreLAB.java | 22 +-
.../hbase/regionserver/MemStoreLABImpl.java | 74 +++--
.../regionserver/TestCompactingMemStore.java | 3 +-
.../regionserver/TestMemStoreChunkPool.java | 6 +-
.../hbase/regionserver/TestMemStoreLAB.java | 8 +-
.../TestMemstoreLABWithoutPool.java | 2 +-
11 files changed, 377 insertions(+), 187 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
index 53458f1..0eebfb5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.yetus.audience.InterfaceAudience;
-
/**
* CellChunkImmutableSegment extends the API supported by a {@link Segment},
* and {@link ImmutableSegment}. This immutable segment is working with CellSet with
@@ -43,6 +42,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
public static final long DEEP_OVERHEAD_CCM =
ImmutableSegment.DEEP_OVERHEAD + ClassSize.CELL_CHUNK_MAP;
+ public static final float INDEX_CHUNK_UNUSED_SPACE_PRECENTAGE = 0.1f;
///////////////////// CONSTRUCTORS /////////////////////
/**------------------------------------------------------------------------
@@ -135,20 +135,12 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
private void initializeCellSet(int numOfCells, MemStoreSegmentsIterator iterator,
MemStoreCompactionStrategy.Action action) {
- // calculate how many chunks we will need for index
- int chunkSize = ChunkCreator.getInstance().getChunkSize();
- int numOfCellsInChunk = CellChunkMap.NUM_OF_CELL_REPS_IN_CHUNK;
- int numberOfChunks = calculateNumberOfChunks(numOfCells, numOfCellsInChunk);
int numOfCellsAfterCompaction = 0;
int currentChunkIdx = 0;
int offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
int numUniqueKeys=0;
Cell prev = null;
- // all index Chunks are allocated from ChunkCreator
- Chunk[] chunks = new Chunk[numberOfChunks];
- for (int i=0; i < numberOfChunks; i++) {
- chunks[i] = this.getMemStoreLAB().getNewExternalChunk();
- }
+ Chunk[] chunks = allocIndexChunks(numOfCells);
while (iterator.hasNext()) { // the iterator hides the elimination logic for compaction
boolean alreadyCopied = false;
Cell c = iterator.next();
@@ -161,7 +153,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
c = copyCellIntoMSLAB(c);
alreadyCopied = true;
}
- if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkSize) {
+ if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunks[currentChunkIdx].size) {
currentChunkIdx++; // continue to the next index chunk
offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
}
@@ -207,15 +199,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
int numOfCells, KeyValueScanner segmentScanner, CellSet oldCellSet,
MemStoreCompactionStrategy.Action action) {
Cell curCell;
- // calculate how many chunks we will need for metadata
- int chunkSize = ChunkCreator.getInstance().getChunkSize();
- int numOfCellsInChunk = CellChunkMap.NUM_OF_CELL_REPS_IN_CHUNK;
- int numberOfChunks = calculateNumberOfChunks(numOfCells, numOfCellsInChunk);
- // all index Chunks are allocated from ChunkCreator
- Chunk[] chunks = new Chunk[numberOfChunks];
- for (int i=0; i < numberOfChunks; i++) {
- chunks[i] = this.getMemStoreLAB().getNewExternalChunk();
- }
+ Chunk[] chunks = allocIndexChunks(numOfCells);
int currentChunkIdx = 0;
int offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
@@ -231,7 +215,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
// are copied into MSLAB here.
curCell = copyCellIntoMSLAB(curCell);
}
- if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkSize) {
+ if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunks[currentChunkIdx].size) {
// continue to the next metadata chunk
currentChunkIdx++;
offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER;
@@ -279,14 +263,58 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
return offset;
}
- private int calculateNumberOfChunks(int numOfCells, int numOfCellsInChunk) {
- int numberOfChunks = numOfCells/numOfCellsInChunk;
- if(numOfCells%numOfCellsInChunk!=0) { // if cells cannot be divided evenly between chunks
+ private int calculateNumberOfChunks(int numOfCells, int chunkSize) {
+ int numOfCellsInChunk = calcNumOfCellsInChunk(chunkSize);
+ int numberOfChunks = numOfCells / numOfCellsInChunk;
+ if(numOfCells % numOfCellsInChunk != 0) { // if cells cannot be divided evenly between chunks
numberOfChunks++; // add one additional chunk
}
return numberOfChunks;
}
+ // Assuming we are going to use regular data chunks as index chunks,
+ // we check here how much free space will remain in the last allocated chunk
+ // (the least occupied one).
+ // If the percentage of its remaining free space is above the INDEX_CHUNK_UNUSED_SPACE
+ // threshold, then we will use index chunks (which are smaller) instead.
+ private ChunkCreator.ChunkType useIndexChunks(int numOfCells) {
+ int dataChunkSize = ChunkCreator.getInstance().getChunkSize();
+ int numOfCellsInChunk = calcNumOfCellsInChunk(dataChunkSize);
+ int cellsInLastChunk = numOfCells % numOfCellsInChunk;
+ if (cellsInLastChunk == 0) { // There is no free space in the last chunk and thus,
+ return ChunkCreator.ChunkType.DATA_CHUNK; // no need to use index chunks.
+ } else {
+ int chunkSpace = dataChunkSize - ChunkCreator.SIZEOF_CHUNK_HEADER;
+ int freeSpaceInLastChunk = chunkSpace - cellsInLastChunk * ClassSize.CELL_CHUNK_MAP_ENTRY;
+ if (freeSpaceInLastChunk > INDEX_CHUNK_UNUSED_SPACE_PRECENTAGE * chunkSpace) {
+ return ChunkCreator.ChunkType.INDEX_CHUNK;
+ }
+ return ChunkCreator.ChunkType.DATA_CHUNK;
+ }
+ }
+
+ private int calcNumOfCellsInChunk(int chunkSize) {
+ int chunkSpace = chunkSize - ChunkCreator.SIZEOF_CHUNK_HEADER;
+ int numOfCellsInChunk = chunkSpace / ClassSize.CELL_CHUNK_MAP_ENTRY;
+ return numOfCellsInChunk;
+ }
+
+ private Chunk[] allocIndexChunks(int numOfCells) {
+ // Decide whether to use regular or small chunks and then
+ // calculate how many chunks we will need for index
+
+ ChunkCreator.ChunkType chunkType = useIndexChunks(numOfCells);
+ int chunkSize = ChunkCreator.getInstance().getChunkSize(chunkType);
+ int numberOfChunks = calculateNumberOfChunks(numOfCells, chunkSize);
+ // all index Chunks are allocated from ChunkCreator
+ Chunk[] chunks = new Chunk[numberOfChunks];
+ // all index Chunks are allocated from ChunkCreator
+ for (int i = 0; i < numberOfChunks; i++) {
+ chunks[i] = this.getMemStoreLAB().getNewExternalChunk(chunkType);
+ }
+ return chunks;
+ }
+
private Cell copyCellIntoMSLAB(Cell cell) {
// Take care for a special case when a cell is copied from on-heap to (probably off-heap) MSLAB.
// The cell allocated as an on-heap JVM object (byte array) occupies slightly different
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java
index a3b45c3..a2c1820 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkMap.java
@@ -57,11 +57,10 @@ public class CellChunkMap extends CellFlatMap {
private final Chunk[] chunks; // the array of chunks, on which the index is based
- // constant number of cell-representations in a chunk
+ // number of cell-representations in a chunk
+ // depends on the size of the chunks (may be index chunks or regular data chunks)
// each chunk starts with its own ID following the cells data
- public static final int NUM_OF_CELL_REPS_IN_CHUNK =
- (ChunkCreator.getInstance().getChunkSize() - ChunkCreator.SIZEOF_CHUNK_HEADER) /
- ClassSize.CELL_CHUNK_MAP_ENTRY;
+ private final int numOfCellRepsInChunk;
/**
* C-tor for creating CellChunkMap from existing Chunk array, which must be ordered
@@ -76,6 +75,12 @@ public class CellChunkMap extends CellFlatMap {
Chunk[] chunks, int min, int max, boolean descending) {
super(comparator, min, max, descending);
this.chunks = chunks;
+ if (chunks != null && chunks.length != 0 && chunks[0] != null) {
+ this.numOfCellRepsInChunk = (chunks[0].size - ChunkCreator.SIZEOF_CHUNK_HEADER) /
+ ClassSize.CELL_CHUNK_MAP_ENTRY;
+ } else { // In case the chunks array was not allocated
+ this.numOfCellRepsInChunk = 0;
+ }
}
/* To be used by base (CellFlatMap) class only to create a sub-CellFlatMap
@@ -89,9 +94,9 @@ public class CellChunkMap extends CellFlatMap {
@Override
protected Cell getCell(int i) {
// get the index of the relevant chunk inside chunk array
- int chunkIndex = (i / NUM_OF_CELL_REPS_IN_CHUNK);
+ int chunkIndex = (i / numOfCellRepsInChunk);
ByteBuffer block = chunks[chunkIndex].getData();// get the ByteBuffer of the relevant chunk
- int j = i - chunkIndex * NUM_OF_CELL_REPS_IN_CHUNK; // get the index of the cell-representation
+ int j = i - chunkIndex * numOfCellRepsInChunk; // get the index of the cell-representation
// find inside the offset inside the chunk holding the index, skip bytes for chunk id
int offsetInBytes = ChunkCreator.SIZEOF_CHUNK_HEADER + j* ClassSize.CELL_CHUNK_MAP_ENTRY;
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
index 199298b..136efee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Chunk.java
@@ -89,6 +89,10 @@ public abstract class Chunk {
return size > ChunkCreator.getInstance().getChunkSize();
}
+ boolean isIndexChunk() {
+ return size == ChunkCreator.getInstance().getChunkSize(ChunkCreator.ChunkType.INDEX_CHUNK);
+ }
+
/**
* Actually claim the memory for this chunk. This should only be called from the thread that
* constructed the chunk. It is thread-safe against other threads calling alloc(), who will block
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java
index b2d4ba8..e62303e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java
@@ -56,27 +56,58 @@ public class ChunkCreator {
// the header size need to be changed in case chunk id size is changed
public static final int SIZEOF_CHUNK_HEADER = Bytes.SIZEOF_INT;
+ /**
+ * Types of chunks, based on their sizes
+ */
+ public enum ChunkType {
+ // An index chunk is a small chunk, allocated from the index chunks pool.
+ // Its size is fixed and is 10% of the size of a data chunk.
+ INDEX_CHUNK,
+ // A data chunk is a regular chunk, allocated from the data chunks pool.
+ // Its size is fixed and given as input to the ChunkCreator c'tor.
+ DATA_CHUNK,
+ // A jumbo chunk isn't allocated from pool. Its size is bigger than the size of a
+ // data chunk, and is determined per chunk (meaning, there is no fixed jumbo size).
+ JUMBO_CHUNK
+ }
+
// mapping from chunk IDs to chunks
private Map<Integer, Chunk> chunkIdMap = new ConcurrentHashMap<Integer, Chunk>();
- private final int chunkSize;
private final boolean offheap;
@VisibleForTesting
- static ChunkCreator INSTANCE;
+ static ChunkCreator instance;
@VisibleForTesting
static boolean chunkPoolDisabled = false;
- private MemStoreChunkPool pool;
+ private MemStoreChunkPool dataChunksPool;
+ private int chunkSize;
+ private MemStoreChunkPool indexChunksPool;
@VisibleForTesting
ChunkCreator(int chunkSize, boolean offheap, long globalMemStoreSize, float poolSizePercentage,
- float initialCountPercentage, HeapMemoryManager heapMemoryManager) {
- this.chunkSize = chunkSize;
+ float initialCountPercentage, HeapMemoryManager heapMemoryManager,
+ float indexChunkSizePercentage) {
this.offheap = offheap;
- this.pool = initializePool(globalMemStoreSize, poolSizePercentage, initialCountPercentage);
- if (heapMemoryManager != null && this.pool != null) {
- // Register with Heap Memory manager
- heapMemoryManager.registerTuneObserver(this.pool);
- }
+ this.chunkSize = chunkSize; // in case pools are not allocated
+ initializePools(chunkSize, globalMemStoreSize, poolSizePercentage, indexChunkSizePercentage,
+ initialCountPercentage, heapMemoryManager);
+ }
+
+ @VisibleForTesting
+ private void initializePools(int chunkSize, long globalMemStoreSize,
+ float poolSizePercentage, float indexChunkSizePercentage,
+ float initialCountPercentage,
+ HeapMemoryManager heapMemoryManager) {
+ this.dataChunksPool = initializePool(globalMemStoreSize,
+ (1 - indexChunkSizePercentage) * poolSizePercentage,
+ initialCountPercentage, chunkSize, heapMemoryManager);
+ // The index chunks pool is needed only when the index type is CCM.
+ // Since the pools are not created at all when the index type isn't CCM,
+ // we don't need to check it here.
+ this.indexChunksPool = initializePool(globalMemStoreSize,
+ indexChunkSizePercentage * poolSizePercentage,
+ initialCountPercentage, (int) (indexChunkSizePercentage * chunkSize),
+ heapMemoryManager);
}
/**
@@ -90,18 +121,30 @@ public class ChunkCreator {
* @return singleton MSLABChunkCreator
*/
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "LI_LAZY_INIT_STATIC",
- justification = "Method is called by single thread at the starting of RS")
+ justification = "Method is called by single thread at the starting of RS")
@VisibleForTesting
public static ChunkCreator initialize(int chunkSize, boolean offheap, long globalMemStoreSize,
- float poolSizePercentage, float initialCountPercentage, HeapMemoryManager heapMemoryManager) {
- if (INSTANCE != null) return INSTANCE;
- INSTANCE = new ChunkCreator(chunkSize, offheap, globalMemStoreSize, poolSizePercentage,
- initialCountPercentage, heapMemoryManager);
- return INSTANCE;
+ float poolSizePercentage, float initialCountPercentage,
+ HeapMemoryManager heapMemoryManager) {
+ if (instance != null) {
+ return instance;
+ }
+ instance = new ChunkCreator(chunkSize, offheap, globalMemStoreSize, poolSizePercentage,
+ initialCountPercentage, heapMemoryManager,
+ MemStoreLABImpl.INDEX_CHUNK_PERCENTAGE_DEFAULT);
+ return instance;
}
static ChunkCreator getInstance() {
- return INSTANCE;
+ return instance;
+ }
+
+ /**
+ * Creates and inits a chunk. The default implementation for a specific chunk size.
+ * @return the chunk that was initialized
+ */
+ Chunk getChunk(ChunkType chunkType) {
+ return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, chunkType);
}
/**
@@ -109,15 +152,37 @@ public class ChunkCreator {
* @return the chunk that was initialized
*/
Chunk getChunk() {
- return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, chunkSize);
+ return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, ChunkType.DATA_CHUNK);
}
/**
- * Creates and inits a chunk. The default implementation for specific index type.
+ * Creates and inits a chunk. The default implementation for a specific index type.
* @return the chunk that was initialized
*/
Chunk getChunk(CompactingMemStore.IndexType chunkIndexType) {
- return getChunk(chunkIndexType, chunkSize);
+ return getChunk(chunkIndexType, ChunkType.DATA_CHUNK);
+ }
+
+ /**
+ * Creates and inits a chunk with specific index type and type.
+ * @return the chunk that was initialized
+ */
+ Chunk getChunk(CompactingMemStore.IndexType chunkIndexType, ChunkType chunkType) {
+ switch (chunkType) {
+ case INDEX_CHUNK:
+ if (indexChunksPool != null) {
+ return getChunk(chunkIndexType, indexChunksPool.getChunkSize());
+ }
+ case DATA_CHUNK:
+ if (dataChunksPool == null) {
+ return getChunk(chunkIndexType, chunkSize);
+ } else {
+ return getChunk(chunkIndexType, dataChunksPool.getChunkSize());
+ }
+ default:
+ throw new IllegalArgumentException(
+ "chunkType must either be INDEX_CHUNK or DATA_CHUNK");
+ }
}
/**
@@ -128,18 +193,28 @@ public class ChunkCreator {
*/
Chunk getChunk(CompactingMemStore.IndexType chunkIndexType, int size) {
Chunk chunk = null;
- // if we have pool and this is not jumbo chunk (when size != chunkSize this is jumbo chunk)
- if ((pool != null) && (size == chunkSize)) {
+ MemStoreChunkPool pool = null;
+
+ // if the size is suitable for one of the pools
+ if (dataChunksPool != null && size == dataChunksPool.getChunkSize()) {
+ pool = dataChunksPool;
+ } else if (indexChunksPool != null && size == indexChunksPool.getChunkSize()) {
+ pool = indexChunksPool;
+ }
+
+ // if we have a pool
+ if (pool != null) {
// the pool creates the chunk internally. The chunk#init() call happens here
- chunk = this.pool.getChunk();
+ chunk = pool.getChunk();
// the pool has run out of maxCount
if (chunk == null) {
if (LOG.isTraceEnabled()) {
- LOG.trace("Chunk pool full (maxCount={}); creating chunk offheap.",
- this.pool.getMaxCount());
+ LOG.trace("The chunk pool is full. Reached maxCount= " + pool.getMaxCount()
+ + ". Creating chunk onheap.");
}
}
}
+
if (chunk == null) {
// the second parameter explains whether CellChunkMap index is requested,
// in that case, put allocated on demand chunk mapping into chunkIdMap
@@ -156,18 +231,18 @@ public class ChunkCreator {
* Creates and inits a chunk of a special size, bigger than a regular chunk size.
* Such a chunk will never come from pool and will always be on demand allocated.
* @return the chunk that was initialized
- * @param chunkIndexType whether the requested chunk is going to be used with CellChunkMap index
* @param jumboSize the special size to be used
*/
- Chunk getJumboChunk(CompactingMemStore.IndexType chunkIndexType, int jumboSize) {
- if (jumboSize <= chunkSize) {
- LOG.warn("Jumbo chunk size=" + jumboSize + " must be more than regular chunk size="
- + chunkSize + "; converting to regular chunk.");
- return getChunk(chunkIndexType,chunkSize);
+ Chunk getJumboChunk(int jumboSize) {
+ int allocSize = jumboSize + SIZEOF_CHUNK_HEADER;
+ if (allocSize <= dataChunksPool.getChunkSize()) {
+ LOG.warn("Jumbo chunk size " + jumboSize + " must be more than regular chunk size "
+ + dataChunksPool.getChunkSize() + ". Converting to regular chunk.");
+ return getChunk(CompactingMemStore.IndexType.CHUNK_MAP);
}
- // the size of the allocation includes
- // both the size requested and a place for the Chunk's header
- return getChunk(chunkIndexType, jumboSize + SIZEOF_CHUNK_HEADER);
+ // the new chunk is going to hold the jumbo cell data and needs to be referenced by
+ // a strong map. Therefore the CCM index type
+ return getChunk(CompactingMemStore.IndexType.CHUNK_MAP, allocSize);
}
/**
@@ -196,21 +271,21 @@ public class ChunkCreator {
// Chunks from pool are created covered with strong references anyway
// TODO: change to CHUNK_MAP if it is generally defined
- private Chunk createChunkForPool() {
- return createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, chunkSize);
+ private Chunk createChunkForPool(CompactingMemStore.IndexType chunkIndexType, int chunkSize) {
+ if (chunkSize != dataChunksPool.getChunkSize() &&
+ chunkSize != indexChunksPool.getChunkSize()) {
+ return null;
+ }
+ return createChunk(true, chunkIndexType, chunkSize);
}
@VisibleForTesting
- // Used to translate the ChunkID into a chunk ref
+ // Used to translate the ChunkID into a chunk ref
Chunk getChunk(int id) {
// can return null if chunk was never mapped
return chunkIdMap.get(id);
}
- int getChunkSize() {
- return this.chunkSize;
- }
-
boolean isOffheap() {
return this.offheap;
}
@@ -224,8 +299,8 @@ public class ChunkCreator {
}
@VisibleForTesting
- // the chunks in the chunkIdMap may already be released so we shouldn't relay
- // on this counting for strong correctness. This method is used only in testing.
+ // the chunks in the chunkIdMap may already be released so we shouldn't relay
+ // on this counting for strong correctness. This method is used only in testing.
int numberOfMappedChunks() {
return this.chunkIdMap.size();
}
@@ -243,6 +318,7 @@ public class ChunkCreator {
* collection on JVM.
*/
private class MemStoreChunkPool implements HeapMemoryTuneObserver {
+ private final int chunkSize;
private int maxCount;
// A queue of reclaimed chunks
@@ -256,21 +332,22 @@ public class ChunkCreator {
private final AtomicLong chunkCount = new AtomicLong();
private final LongAdder reusedChunkCount = new LongAdder();
- MemStoreChunkPool(int maxCount, int initialCount, float poolSizePercentage) {
+ MemStoreChunkPool(int chunkSize, int maxCount, int initialCount, float poolSizePercentage) {
+ this.chunkSize = chunkSize;
this.maxCount = maxCount;
this.poolSizePercentage = poolSizePercentage;
this.reclaimedChunks = new LinkedBlockingQueue<>();
for (int i = 0; i < initialCount; i++) {
- Chunk chunk = createChunkForPool();
+ Chunk chunk = createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, chunkSize);
chunk.init();
reclaimedChunks.add(chunk);
}
chunkCount.set(initialCount);
final String n = Thread.currentThread().getName();
scheduleThreadPool = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder()
- .setNameFormat(n + "-MemStoreChunkPool Statistics").setDaemon(true).build());
+ .setNameFormat(n + "-MemStoreChunkPool Statistics").setDaemon(true).build());
this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(), statThreadPeriod,
- statThreadPeriod, TimeUnit.SECONDS);
+ statThreadPeriod, TimeUnit.SECONDS);
}
/**
@@ -283,6 +360,10 @@ public class ChunkCreator {
* @see #putbackChunks(Chunk)
*/
Chunk getChunk() {
+ return getChunk(CompactingMemStore.IndexType.ARRAY_MAP);
+ }
+
+ Chunk getChunk(CompactingMemStore.IndexType chunkIndexType) {
Chunk chunk = reclaimedChunks.poll();
if (chunk != null) {
chunk.reset();
@@ -293,7 +374,7 @@ public class ChunkCreator {
long created = this.chunkCount.get();
if (created < this.maxCount) {
if (this.chunkCount.compareAndSet(created, created + 1)) {
- chunk = createChunkForPool();
+ chunk = createChunkForPool(chunkIndexType, chunkSize);
break;
}
} else {
@@ -304,6 +385,10 @@ public class ChunkCreator {
return chunk;
}
+ int getChunkSize() {
+ return chunkSize;
+ }
+
/**
* Add the chunks to the pool, when the pool achieves the max size, it will skip the remaining
* chunks
@@ -311,7 +396,7 @@ public class ChunkCreator {
*/
private void putbackChunks(Chunk c) {
int toAdd = this.maxCount - reclaimedChunks.size();
- if (c.isFromPool() && toAdd > 0) {
+ if (c.isFromPool() && c.size == chunkSize && toAdd > 0) {
reclaimedChunks.add(c);
} else {
// remove the chunk (that is not going to pool)
@@ -336,10 +421,11 @@ public class ChunkCreator {
long created = chunkCount.get();
long reused = reusedChunkCount.sum();
long total = created + reused;
- LOG.debug("Stats: current pool size=" + reclaimedChunks.size()
- + ",created chunk count=" + created
- + ",reused chunk count=" + reused
- + ",reuseRatio=" + (total == 0 ? "0" : StringUtils.formatPercent(
+ LOG.debug("Stats (chunk size=" + chunkSize + "): "
+ + "current pool size=" + reclaimedChunks.size()
+ + ",created chunk count=" + created
+ + ",reused chunk count=" + reused
+ + ",reuseRatio=" + (total == 0 ? "0" : StringUtils.formatPercent(
(float) reused / (float) total, 2)));
}
}
@@ -356,7 +442,7 @@ public class ChunkCreator {
return;
}
int newMaxCount =
- (int) (newMemstoreSize * poolSizePercentage / getChunkSize());
+ (int) (newMemstoreSize * poolSizePercentage / getChunkSize());
if (newMaxCount != this.maxCount) {
// We need an adjustment in the chunks numbers
if (newMaxCount > this.maxCount) {
@@ -387,7 +473,8 @@ public class ChunkCreator {
}
private MemStoreChunkPool initializePool(long globalMemStoreSize, float poolSizePercentage,
- float initialCountPercentage) {
+ float initialCountPercentage, int chunkSize,
+ HeapMemoryManager heapMemoryManager) {
if (poolSizePercentage <= 0) {
LOG.info("PoolSizePercentage is less than 0. So not using pool");
return null;
@@ -397,47 +484,90 @@ public class ChunkCreator {
}
if (poolSizePercentage > 1.0) {
throw new IllegalArgumentException(
- MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0");
+ MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0");
}
- int maxCount = (int) (globalMemStoreSize * poolSizePercentage / getChunkSize());
+ int maxCount = (int) (globalMemStoreSize * poolSizePercentage / chunkSize);
if (initialCountPercentage > 1.0 || initialCountPercentage < 0) {
throw new IllegalArgumentException(
- MemStoreLAB.CHUNK_POOL_INITIALSIZE_KEY + " must be between 0.0 and 1.0");
+ MemStoreLAB.CHUNK_POOL_INITIALSIZE_KEY + " must be between 0.0 and 1.0");
}
int initialCount = (int) (initialCountPercentage * maxCount);
- LOG.info("Allocating MemStoreChunkPool with chunk size="
- + StringUtils.byteDesc(getChunkSize()) + ", max count=" + maxCount
- + ", initial count=" + initialCount);
- return new MemStoreChunkPool(maxCount, initialCount, poolSizePercentage);
+ LOG.info("Allocating MemStoreChunkPool with chunk size "
+ + StringUtils.byteDesc(chunkSize) + ", max count " + maxCount
+ + ", initial count " + initialCount);
+ MemStoreChunkPool memStoreChunkPool = new MemStoreChunkPool(chunkSize, maxCount,
+ initialCount, poolSizePercentage);
+ if (heapMemoryManager != null && memStoreChunkPool != null) {
+ // Register with Heap Memory manager
+ heapMemoryManager.registerTuneObserver(memStoreChunkPool);
+ }
+ return memStoreChunkPool;
}
@VisibleForTesting
int getMaxCount() {
- if (pool != null) {
- return pool.getMaxCount();
+ return getMaxCount(ChunkType.DATA_CHUNK);
+ }
+
+ @VisibleForTesting
+ int getMaxCount(ChunkType chunkType) {
+ switch (chunkType) {
+ case INDEX_CHUNK:
+ if (indexChunksPool != null) {
+ return indexChunksPool.getMaxCount();
+ }
+ break;
+ case DATA_CHUNK:
+ if (dataChunksPool != null) {
+ return dataChunksPool.getMaxCount();
+ }
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "chunkType must either be INDEX_CHUNK or DATA_CHUNK");
}
+
return 0;
}
@VisibleForTesting
int getPoolSize() {
- if (pool != null) {
- return pool.reclaimedChunks.size();
+ return getPoolSize(ChunkType.DATA_CHUNK);
+ }
+
+ @VisibleForTesting
+ int getPoolSize(ChunkType chunkType) {
+ switch (chunkType) {
+ case INDEX_CHUNK:
+ if (indexChunksPool != null) {
+ return indexChunksPool.reclaimedChunks.size();
+ }
+ break;
+ case DATA_CHUNK:
+ if (dataChunksPool != null) {
+ return dataChunksPool.reclaimedChunks.size();
+ }
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "chunkType must either be INDEX_CHUNK or DATA_CHUNK");
}
return 0;
}
@VisibleForTesting
boolean isChunkInPool(int chunkId) {
- if (pool != null) {
- // chunks that are from pool will return true chunk reference not null
- Chunk c = getChunk(chunkId);
- if (c==null) {
- return false;
- }
- return pool.reclaimedChunks.contains(c);
+ Chunk c = getChunk(chunkId);
+ if (c==null) {
+ return false;
}
+ // chunks that are from pool will return true chunk reference not null
+ if (dataChunksPool != null && dataChunksPool.reclaimedChunks.contains(c)) {
+ return true;
+ } else if (indexChunksPool != null && indexChunksPool.reclaimedChunks.contains(c)) {
+ return true;
+ }
return false;
}
@@ -446,31 +576,58 @@ public class ChunkCreator {
*/
@VisibleForTesting
void clearChunksInPool() {
- if (pool != null) {
- pool.reclaimedChunks.clear();
+ if (dataChunksPool != null) {
+ dataChunksPool.reclaimedChunks.clear();
+ }
+ if (indexChunksPool != null) {
+ indexChunksPool.reclaimedChunks.clear();
+ }
+ }
+
+ int getChunkSize() {
+ return getChunkSize(ChunkType.DATA_CHUNK);
+ }
+
+ int getChunkSize(ChunkType chunkType) {
+ switch (chunkType) {
+ case INDEX_CHUNK:
+ if (indexChunksPool != null) {
+ return indexChunksPool.getChunkSize();
+ }
+ case DATA_CHUNK:
+ if (dataChunksPool != null) {
+ return dataChunksPool.getChunkSize();
+ } else { // When pools are empty
+ return chunkSize;
+ }
+ default:
+ throw new IllegalArgumentException(
+ "chunkType must either be INDEX_CHUNK or DATA_CHUNK");
}
}
synchronized void putbackChunks(Set<Integer> chunks) {
// if there is no pool just try to clear the chunkIdMap in case there is something
- if ( pool == null ) {
+ if (dataChunksPool == null && indexChunksPool == null) {
this.removeChunks(chunks);
return;
}
- // if there is pool, go over all chunk IDs that came back, the chunks may be from pool or not
+ // if there is a pool, go over all chunk IDs that came back, the chunks may be from pool or not
for (int chunkID : chunks) {
// translate chunk ID to chunk, if chunk initially wasn't in pool
// this translation will (most likely) return null
Chunk chunk = ChunkCreator.this.getChunk(chunkID);
if (chunk != null) {
- // Jumbo chunks are covered with chunkIdMap, but are not from pool, so such a chunk should
- // be released here without going to pool.
- // Removing them from chunkIdMap will cause their removal by the GC.
- if (chunk.isJumbo()) {
- this.removeChunk(chunkID);
+ if (chunk.isFromPool() && chunk.isIndexChunk()) {
+ indexChunksPool.putbackChunks(chunk);
+ } else if (chunk.isFromPool() && chunk.size == dataChunksPool.getChunkSize()) {
+ dataChunksPool.putbackChunks(chunk);
} else {
- pool.putbackChunks(chunk);
+ // chunks which are not from one of the pools
+ // should be released without going to the pools.
+ // Removing them from chunkIdMap will cause their removal by the GC.
+ this.removeChunk(chunkID);
}
}
// if chunk is null, it was never covered by the chunkIdMap (and so wasn't in pool also),
@@ -480,3 +637,4 @@ public class ChunkCreator {
}
}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
index 71648a0..0ce52b6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java
@@ -61,32 +61,26 @@ public class ImmutableMemStoreLAB implements MemStoreLAB {
return mslab.forceCopyOfBigCellInto(cell);
}
- /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array.
- ** Returning a new chunk, without replacing current chunk,
+ /* Returning a new pool chunk, without replacing current chunk,
** meaning MSLABImpl does not make the returned chunk as CurChunk.
** The space on this chunk will be allocated externally.
- ** The interface is only for external callers
+ ** The interface is only for external callers.
*/
@Override
- public Chunk getNewExternalChunk() {
+ public Chunk getNewExternalChunk(ChunkCreator.ChunkType chunkType) {
MemStoreLAB mslab = this.mslabs.get(0);
- return mslab.getNewExternalChunk();
+ return mslab.getNewExternalChunk(chunkType);
}
- /* Creating chunk to be used as data chunk in CellChunkMap.
- ** This chunk is bigger the normal constant chunk size, and thus called JumboChunk it is used for
- ** jumbo cells (which size is bigger than normal chunks).
- ** Jumbo Chunks are needed only for CCM and thus are created only in
- ** CompactingMemStore.IndexType.CHUNK_MAP type.
- ** Returning a new chunk, without replacing current chunk,
- ** meaning MSLABImpl does not make the returned chunk as CurChunk.
- ** The space on this chunk will be allocated externally.
- ** The interface is only for external callers
- */
+ /* Returning a new chunk, without replacing current chunk,
+ ** meaning MSLABImpl does not make the returned chunk as CurChunk.
+ ** The space on this chunk will be allocated externally.
+ ** The interface is only for external callers.
+ */
@Override
- public Chunk getNewExternalJumboChunk(int size) {
+ public Chunk getNewExternalChunk(int size) {
MemStoreLAB mslab = this.mslabs.get(0);
- return mslab.getNewExternalJumboChunk(size);
+ return mslab.getNewExternalChunk(size);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
index 8b77981..90cf932 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java
@@ -52,6 +52,8 @@ public interface MemStoreLAB {
String CHUNK_SIZE_KEY = "hbase.hregion.memstore.mslab.chunksize";
int CHUNK_SIZE_DEFAULT = 2048 * 1024;
+ String INDEX_CHUNK_PERCENTAGE_KEY = "hbase.hregion.memstore.mslab.indexchunksize";
+ float INDEX_CHUNK_PERCENTAGE_DEFAULT = 0.1f;
String MAX_ALLOC_KEY = "hbase.hregion.memstore.mslab.max.allocation";
int MAX_ALLOC_DEFAULT = 256 * 1024; // allocs bigger than this don't go through
// allocator
@@ -94,25 +96,19 @@ public interface MemStoreLAB {
*/
void decScannerCount();
- /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array.
- ** Returning a new chunk, without replacing current chunk,
+ /* Returning a new pool chunk, without replacing current chunk,
** meaning MSLABImpl does not make the returned chunk as CurChunk.
** The space on this chunk will be allocated externally.
- ** The interface is only for external callers
+ ** The interface is only for external callers.
*/
- Chunk getNewExternalChunk();
-
- /* Creating chunk to be used as data chunk in CellChunkMap.
- ** This chunk is bigger than normal constant chunk size, and thus called JumboChunk it is used for
- ** jumbo cells (which size is bigger than normal chunks).
- ** Jumbo Chunks are needed only for CCM and thus are created only in
- ** CompactingMemStore.IndexType.CHUNK_MAP type.
- ** Returning a new chunk, without replacing current chunk,
+ Chunk getNewExternalChunk(ChunkCreator.ChunkType chunkType);
+
+ /* Returning a new chunk, without replacing current chunk,
** meaning MSLABImpl does not make the returned chunk as CurChunk.
** The space on this chunk will be allocated externally.
- ** The interface is only for external callers
+ ** The interface is only for external callers.
*/
- Chunk getNewExternalJumboChunk(int size);
+ Chunk getNewExternalChunk(int size);
static MemStoreLAB newInstance(Configuration conf) {
MemStoreLAB memStoreLAB = null;
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
index 4ff0480..0db0fd9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
@@ -67,14 +67,14 @@ public class MemStoreLABImpl implements MemStoreLAB {
static final Logger LOG = LoggerFactory.getLogger(MemStoreLABImpl.class);
- private AtomicReference<Chunk> curChunk = new AtomicReference<>();
+ private AtomicReference<Chunk> currChunk = new AtomicReference<>();
// Lock to manage multiple handlers requesting for a chunk
private ReentrantLock lock = new ReentrantLock();
// A set of chunks contained by this memstore LAB
@VisibleForTesting
Set<Integer> chunks = new ConcurrentSkipListSet<Integer>();
- private final int chunkSize;
+ private final int dataChunkSize;
private final int maxAlloc;
private final ChunkCreator chunkCreator;
private final CompactingMemStore.IndexType idxType; // what index is used for corresponding segment
@@ -94,11 +94,11 @@ public class MemStoreLABImpl implements MemStoreLAB {
}
public MemStoreLABImpl(Configuration conf) {
- chunkSize = conf.getInt(CHUNK_SIZE_KEY, CHUNK_SIZE_DEFAULT);
+ dataChunkSize = conf.getInt(CHUNK_SIZE_KEY, CHUNK_SIZE_DEFAULT);
maxAlloc = conf.getInt(MAX_ALLOC_KEY, MAX_ALLOC_DEFAULT);
this.chunkCreator = ChunkCreator.getInstance();
// if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one!
- Preconditions.checkArgument(maxAlloc <= chunkSize,
+ Preconditions.checkArgument(maxAlloc <= dataChunkSize,
MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY);
// if user requested to work with MSLABs (whether on- or off-heap), then the
@@ -120,14 +120,14 @@ public class MemStoreLABImpl implements MemStoreLAB {
*/
@Override
public Cell forceCopyOfBigCellInto(Cell cell) {
- int size = KeyValueUtil.length(cell);
+ int size = KeyValueUtil.length(cell) + ChunkCreator.SIZEOF_CHUNK_HEADER;
Preconditions.checkArgument(size >= 0, "negative size");
- if (size <= chunkSize) {
+ if (size <= dataChunkSize) {
// Using copyCellInto for cells which are bigger than the original maxAlloc
- Cell newCell = copyCellInto(cell, chunkSize);
+ Cell newCell = copyCellInto(cell, dataChunkSize);
return newCell;
} else {
- Chunk c = getNewExternalJumboChunk(size);
+ Chunk c = getNewExternalChunk(size);
int allocOffset = c.alloc(size);
return copyToChunkCell(cell, c.getData(), allocOffset, size);
}
@@ -240,7 +240,7 @@ public class MemStoreLABImpl implements MemStoreLAB {
* @return true if we won the race to retire the chunk
*/
private void tryRetireChunk(Chunk c) {
- curChunk.compareAndSet(c, null);
+ currChunk.compareAndSet(c, null);
// If the CAS succeeds, that means that we won the race
// to retire the chunk. We could use this opportunity to
// update metrics on external fragmentation.
@@ -255,7 +255,8 @@ public class MemStoreLABImpl implements MemStoreLAB {
*/
private Chunk getOrMakeChunk() {
// Try to get the chunk
- Chunk c = curChunk.get();
+ Chunk c;
+ c = currChunk.get();
if (c != null) {
return c;
}
@@ -265,14 +266,14 @@ public class MemStoreLABImpl implements MemStoreLAB {
if (lock.tryLock()) {
try {
// once again check inside the lock
- c = curChunk.get();
+ c = currChunk.get();
if (c != null) {
return c;
}
c = this.chunkCreator.getChunk(idxType);
if (c != null) {
// set the curChunk. No need of CAS as only one thread will be here
- curChunk.set(c);
+ currChunk.set(c);
chunks.add(c.getId());
return c;
}
@@ -283,38 +284,41 @@ public class MemStoreLABImpl implements MemStoreLAB {
return null;
}
- /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array.
- ** Returning a new chunk, without replacing current chunk,
+ /* Returning a new pool chunk, without replacing current chunk,
** meaning MSLABImpl does not make the returned chunk as CurChunk.
** The space on this chunk will be allocated externally.
- ** The interface is only for external callers
+ ** The interface is only for external callers.
*/
@Override
- public Chunk getNewExternalChunk() {
- // the new chunk is going to be part of the chunk array and will always be referenced
- Chunk c = this.chunkCreator.getChunk();
- chunks.add(c.getId());
- return c;
+ public Chunk getNewExternalChunk(ChunkCreator.ChunkType chunkType) {
+ switch (chunkType) {
+ case INDEX_CHUNK:
+ case DATA_CHUNK:
+ Chunk c = this.chunkCreator.getChunk(chunkType);
+ chunks.add(c.getId());
+ return c;
+ case JUMBO_CHUNK: // a jumbo chunk doesn't have a fixed size
+ default:
+ return null;
+ }
}
- /* Creating chunk to be used as data chunk in CellChunkMap.
- ** This chunk is bigger than normal constant chunk size, and thus called JumboChunk.
- ** JumboChunk is used for jumbo cell (which size is bigger than normal chunk). It is allocated
- ** once per cell. So even if there is space this is not reused.
- ** Jumbo Chunks are used only for CCM and thus are created only in
- ** CompactingMemStore.IndexType.CHUNK_MAP type.
- ** Returning a new chunk, without replacing current chunk,
+ /* Returning a new chunk, without replacing current chunk,
** meaning MSLABImpl does not make the returned chunk as CurChunk.
** The space on this chunk will be allocated externally.
- ** The interface is only for external callers
+ ** The interface is only for external callers.
+ ** Chunks from pools are not allocated from here, since they have fixed sizes
*/
@Override
- public Chunk getNewExternalJumboChunk(int size) {
- // the new chunk is going to hold the jumbo cell data and need to be referenced by a strong map
- // thus giving the CCM index type
- Chunk c = this.chunkCreator.getJumboChunk(CompactingMemStore.IndexType.CHUNK_MAP, size);
- chunks.add(c.getId());
- return c;
+ public Chunk getNewExternalChunk(int size) {
+ int allocSize = size + ChunkCreator.getInstance().SIZEOF_CHUNK_HEADER;
+ if (allocSize <= ChunkCreator.getInstance().getChunkSize()) {
+ return getNewExternalChunk(ChunkCreator.ChunkType.DATA_CHUNK);
+ } else {
+ Chunk c = this.chunkCreator.getJumboChunk(size);
+ chunks.add(c.getId());
+ return c;
+ }
}
@Override
@@ -329,7 +333,7 @@ public class MemStoreLABImpl implements MemStoreLAB {
@VisibleForTesting
Chunk getCurrentChunk() {
- return this.curChunk.get();
+ return currChunk.get();
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
index 505c2f0..5cbfff9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
@@ -118,7 +118,8 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
long globalMemStoreLimit = (long) (ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
.getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false));
chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false,
- globalMemStoreLimit, 0.4f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null);
+ globalMemStoreLimit, 0.4f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT,
+ null);
assertTrue(chunkCreator != null);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
index 891e835..4f3de36 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
@@ -233,10 +233,10 @@ public class TestMemStoreChunkPool {
final int chunkSize = 40;
final int valSize = 7;
ChunkCreator oldCreator = ChunkCreator.getInstance();
- ChunkCreator newCreator = new ChunkCreator(chunkSize, false, 400, 1, 0.5f, null);
+ ChunkCreator newCreator = new ChunkCreator(chunkSize, false, 400, 1, 0.5f, null, 0);
assertEquals(initialCount, newCreator.getPoolSize());
assertEquals(maxCount, newCreator.getMaxCount());
- ChunkCreator.INSTANCE = newCreator;// Replace the global ref with the new one we created.
+ ChunkCreator.instance = newCreator;// Replace the global ref with the new one we created.
// Used it for the testing. Later in finally we put
// back the original
final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"),
@@ -265,7 +265,7 @@ public class TestMemStoreChunkPool {
t3.join();
assertTrue(newCreator.getPoolSize() <= maxCount);
} finally {
- ChunkCreator.INSTANCE = oldCreator;
+ ChunkCreator.instance = oldCreator;
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
index 52491e7..ef4ad69 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java
@@ -206,8 +206,8 @@ public class TestMemStoreLAB {
MemStoreLABImpl mslab = new MemStoreLABImpl();
// by default setting, there should be no chunks initialized in the pool
assertTrue(mslab.getPooledChunks().isEmpty());
- oldInstance = ChunkCreator.INSTANCE;
- ChunkCreator.INSTANCE = null;
+ oldInstance = ChunkCreator.instance;
+ ChunkCreator.instance = null;
// reset mslab with chunk pool
Configuration conf = HBaseConfiguration.create();
conf.setDouble(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.1);
@@ -251,7 +251,7 @@ public class TestMemStoreLAB {
}
// none of the chunkIds would have been returned back
assertTrue("All the chunks must have been cleared",
- ChunkCreator.INSTANCE.numberOfMappedChunks() != 0);
+ ChunkCreator.instance.numberOfMappedChunks() != 0);
int pooledChunksNum = mslab.getPooledChunks().size();
// close the mslab
mslab.close();
@@ -261,7 +261,7 @@ public class TestMemStoreLAB {
+ " after mslab closed but actually: " + (pooledChunksNum-queueLength),
pooledChunksNum-queueLength == 0);
} finally {
- ChunkCreator.INSTANCE = oldInstance;
+ ChunkCreator.instance = oldInstance;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/92d04d57/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java
index 0813a8c..e2da5d0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemstoreLABWithoutPool.java
@@ -148,7 +148,7 @@ public class TestMemstoreLABWithoutPool {
}
// all of the chunkIds would have been returned back
assertTrue("All the chunks must have been cleared",
- ChunkCreator.INSTANCE.numberOfMappedChunks() == 0);
+ ChunkCreator.instance.numberOfMappedChunks() == 0);
}
private Thread getChunkQueueTestThread(final MemStoreLABImpl mslab, String threadName,