You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by la...@apache.org on 2015/06/30 03:10:31 UTC
[3/4] hbase git commit: HBASE-13959 Region splitting uses a single
thread in most common cases. (Hari Krishna Dara)
HBASE-13959 Region splitting uses a single thread in most common cases. (Hari Krishna Dara)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/38f43acc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/38f43acc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/38f43acc
Branch: refs/heads/branch-1.2
Commit: 38f43acc4ca1a1f57793ae906c911e40904b6249
Parents: 39de65d
Author: Lars Hofhansl <la...@apache.org>
Authored: Mon Jun 29 18:08:46 2015 -0700
Committer: Lars Hofhansl <la...@apache.org>
Committed: Mon Jun 29 18:10:25 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/HConstants.java | 7 +++++
.../regionserver/SplitTransactionImpl.java | 30 ++++++++++++++++----
2 files changed, 32 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/38f43acc/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index e349912..fb5bd31 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1174,6 +1174,13 @@ public final class HConstants {
"hbase.heap.occupancy.high_water_mark";
public static final float DEFAULT_HEAP_OCCUPANCY_HIGH_WATERMARK = 0.98f;
+ /**
+ * The max number of threads used for splitting storefiles in parallel during
+ * the region split process.
+ */
+ public static final String REGION_SPLIT_THREADS_MAX =
+ "hbase.regionserver.region.split.threads.max";
+
private HConstants() {
// Can't be instantiated with this ctor.
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/38f43acc/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
index 08212a3..5c24eaf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransactionImpl.java
@@ -682,17 +682,29 @@ public class SplitTransactionImpl implements SplitTransaction {
// The following code sets up a thread pool executor with as many slots as
// there's files to split. It then fires up everything, waits for
// completion and finally checks for any exception
- int nbFiles = hstoreFilesToSplit.size();
+ int nbFiles = 0;
+ for (Map.Entry<byte[], List<StoreFile>> entry: hstoreFilesToSplit.entrySet()) {
+ nbFiles += entry.getValue().size();
+ }
if (nbFiles == 0) {
// no file needs to be splitted.
return new Pair<Integer, Integer>(0,0);
}
- LOG.info("Preparing to split " + nbFiles + " storefiles for region " + this.parent);
+ // Default max #threads to use is the smaller of table's configured number of blocking store
+ // files or the available number of logical cores.
+ int defMaxThreads = Math.min(parent.conf.getInt(HStore.BLOCKING_STOREFILES_KEY,
+ HStore.DEFAULT_BLOCKING_STOREFILE_COUNT),
+ Runtime.getRuntime().availableProcessors());
+ // Max #threads is the smaller of the number of storefiles or the default max determined above.
+ int maxThreads = Math.min(parent.conf.getInt(HConstants.REGION_SPLIT_THREADS_MAX,
+ defMaxThreads), nbFiles);
+ LOG.info("Preparing to split " + nbFiles + " storefiles for region " + this.parent +
+ " using " + maxThreads + " threads");
ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
builder.setNameFormat("StoreFileSplitter-%1$d");
ThreadFactory factory = builder.build();
ThreadPoolExecutor threadPool =
- (ThreadPoolExecutor) Executors.newFixedThreadPool(nbFiles, factory);
+ (ThreadPoolExecutor) Executors.newFixedThreadPool(maxThreads, factory);
List<Future<Pair<Path,Path>>> futures = new ArrayList<Future<Pair<Path,Path>>> (nbFiles);
// Split each store file.
@@ -738,14 +750,18 @@ public class SplitTransactionImpl implements SplitTransaction {
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Split storefiles for region " + this.parent + " Daugther A: " + created_a
- + " storefiles, Daugther B: " + created_b + " storefiles.");
+ LOG.debug("Split storefiles for region " + this.parent + " Daughter A: " + created_a
+ + " storefiles, Daughter B: " + created_b + " storefiles.");
}
return new Pair<Integer, Integer>(created_a, created_b);
}
private Pair<Path, Path> splitStoreFile(final byte[] family, final StoreFile sf)
throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Splitting started for store file: " + sf.getPath() + " for region: " +
+ this.parent);
+ }
HRegionFileSystem fs = this.parent.getRegionFileSystem();
String familyName = Bytes.toString(family);
@@ -755,6 +771,10 @@ public class SplitTransactionImpl implements SplitTransaction {
Path path_b =
fs.splitStoreFile(this.hri_b, familyName, sf, this.splitrow, true,
this.parent.getSplitPolicy());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Splitting complete for store file: " + sf.getPath() + " for region: " +
+ this.parent);
+ }
return new Pair<Path,Path>(path_a, path_b);
}