You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/01/16 11:27:03 UTC
[1/4] hbase git commit: HBASE-12773 Add warning message when user is
trying to bulkload a large HFile (Srikanth Srungarapu)
Repository: hbase
Updated Branches:
refs/heads/0.98 5e36e70cf -> 358e60027
refs/heads/branch-1 2e08bb3b4 -> 52b4db860
refs/heads/branch-1.0 f87aab7ec -> 241ad0d76
refs/heads/master eafc07a06 -> 621b33f2f
HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/621b33f2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/621b33f2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/621b33f2
Branch: refs/heads/master
Commit: 621b33f2f9677bd30c78de0ada757b4349eb017b
Parents: eafc07a
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Thu Jan 15 23:13:47 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Thu Jan 15 23:13:47 2015 +0000
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 11 +++++++++--
.../org/apache/hadoop/hbase/regionserver/HStore.java | 6 ++++++
2 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/621b33f2/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 2115978..45f57b3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -206,8 +206,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
}
Path familyDir = stat.getPath();
byte[] family = familyDir.getName().getBytes();
- Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
- for (Path hfile : hfiles) {
+ FileStatus[] hfileStatuses = fs.listStatus(familyDir);
+ for (FileStatus hfileStatus : hfileStatuses) {
+ long length = hfileStatus.getLen();
+ Path hfile = hfileStatus.getPath();
// Skip "_", reference, HFileLink
String fileName = hfile.getName();
if (fileName.startsWith("_")) continue;
@@ -219,6 +221,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LOG.warn("Skipping HFileLink " + fileName);
continue;
}
+ if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
+ HConstants.DEFAULT_MAX_FILE_SIZE)) {
+ LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " +
+ length + " bytes can be problematic as it may lead to oversplitting.");
+ }
ret.add(new LoadQueueItem(family, hfile));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/621b33f2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index b674fea..047d689 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -709,6 +709,12 @@ public class HStore implements Store {
+ this.getRegionInfo().getRegionNameAsString());
}
+ if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
+ HConstants.DEFAULT_MAX_FILE_SIZE)) {
+ LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
+ reader.length() + " bytes can be problematic as it may lead to oversplitting.");
+ }
+
if (verifyBulkLoads) {
long verificationStartTime = EnvironmentEdgeManager.currentTime();
LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
[2/4] hbase git commit: HBASE-12773 Add warning message when user is
trying to bulkload a large HFile (Srikanth Srungarapu)
Posted by mb...@apache.org.
HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/52b4db86
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/52b4db86
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/52b4db86
Branch: refs/heads/branch-1
Commit: 52b4db860780fea6cf5ec0dabb34a581090a535d
Parents: 2e08bb3
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Thu Jan 15 23:13:47 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Thu Jan 15 23:14:29 2015 +0000
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 11 +++++++++--
.../org/apache/hadoop/hbase/regionserver/HStore.java | 6 ++++++
2 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/52b4db86/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index b6e8d9c..9903fe9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -206,8 +206,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
}
Path familyDir = stat.getPath();
byte[] family = familyDir.getName().getBytes();
- Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
- for (Path hfile : hfiles) {
+ FileStatus[] hfileStatuses = fs.listStatus(familyDir);
+ for (FileStatus hfileStatus : hfileStatuses) {
+ long length = hfileStatus.getLen();
+ Path hfile = hfileStatus.getPath();
// Skip "_", reference, HFileLink
String fileName = hfile.getName();
if (fileName.startsWith("_")) continue;
@@ -219,6 +221,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LOG.warn("Skipping HFileLink " + fileName);
continue;
}
+ if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
+ HConstants.DEFAULT_MAX_FILE_SIZE)) {
+ LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " +
+ length + " bytes can be problematic as it may lead to oversplitting.");
+ }
ret.add(new LoadQueueItem(family, hfile));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/52b4db86/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index ad701b7..6a65038 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -709,6 +709,12 @@ public class HStore implements Store {
+ this.getRegionInfo().getRegionNameAsString());
}
+ if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
+ HConstants.DEFAULT_MAX_FILE_SIZE)) {
+ LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
+ reader.length() + " bytes can be problematic as it may lead to oversplitting.");
+ }
+
if (verifyBulkLoads) {
long verificationStartTime = EnvironmentEdgeManager.currentTime();
LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
[4/4] hbase git commit: HBASE-12773 Add warning message when user is
trying to bulkload a large HFile (Srikanth Srungarapu)
Posted by mb...@apache.org.
HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/358e6002
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/358e6002
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/358e6002
Branch: refs/heads/0.98
Commit: 358e60027502f7534a707091d7e53081cbfeb824
Parents: 5e36e70
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Fri Jan 16 10:24:33 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Fri Jan 16 10:24:33 2015 +0000
----------------------------------------------------------------------
.../hbase/mapreduce/LoadIncrementalHFiles.java | 22 ++++++++++++--------
.../hadoop/hbase/regionserver/HStore.java | 9 ++++++--
2 files changed, 20 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/358e6002/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 1d8282d..cf7a92c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -36,6 +36,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
+import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -43,20 +44,17 @@ import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.mutable.MutableInt;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
@@ -64,6 +62,8 @@ import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HTable;
@@ -71,7 +71,6 @@ import org.apache.hadoop.hbase.client.RegionServerCallable;
import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
import org.apache.hadoop.hbase.client.coprocessor.SecureBulkLoadClient;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.HalfStoreFileReader;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
@@ -97,8 +96,6 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import java.util.UUID;
-
/**
* Tool to load the output of HFileOutputFormat into an existing table.
* @see #usage()
@@ -207,9 +204,16 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
continue;
}
byte[] family = familyDir.getName().getBytes();
- Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
- for (Path hfile : hfiles) {
+ FileStatus[] hfileStatuses = fs.listStatus(familyDir);
+ for (FileStatus hfileStatus : hfileStatuses) {
+ long length = hfileStatus.getLen();
+ Path hfile = hfileStatus.getPath();
if (hfile.getName().startsWith("_")) continue;
+ if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
+ HConstants.DEFAULT_MAX_FILE_SIZE)) {
+ LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " +
+ length + " bytes can be problematic as it may lead to oversplitting.");
+ }
ret.add(new LoadQueueItem(family, hfile));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/358e6002/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 79942d4..f6492aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
-import java.io.FileNotFoundException;
import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
import java.security.Key;
@@ -45,7 +44,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -60,6 +58,7 @@ import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.TagType;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.crypto.Cipher;
@@ -645,6 +644,12 @@ public class HStore implements Store {
+ this.getRegionInfo().getRegionNameAsString());
}
+ if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
+ HConstants.DEFAULT_MAX_FILE_SIZE)) {
+ LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
+ reader.length() + " bytes can be problematic as it may lead to oversplitting.");
+ }
+
if (verifyBulkLoads) {
long verificationStartTime = EnvironmentEdgeManager.currentTimeMillis();
LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());
[3/4] hbase git commit: HBASE-12773 Add warning message when user is
trying to bulkload a large HFile (Srikanth Srungarapu)
Posted by mb...@apache.org.
HBASE-12773 Add warning message when user is trying to bulkload a large HFile (Srikanth Srungarapu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/241ad0d7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/241ad0d7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/241ad0d7
Branch: refs/heads/branch-1.0
Commit: 241ad0d767d0d53d8777ca2f4e77ff04c4b04216
Parents: f87aab7
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Thu Jan 15 23:13:47 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Thu Jan 15 23:15:40 2015 +0000
----------------------------------------------------------------------
.../hadoop/hbase/mapreduce/LoadIncrementalHFiles.java | 11 +++++++++--
.../org/apache/hadoop/hbase/regionserver/HStore.java | 6 ++++++
2 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/241ad0d7/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index b4b6adc..55d20f4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -204,8 +204,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
}
Path familyDir = stat.getPath();
byte[] family = familyDir.getName().getBytes();
- Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
- for (Path hfile : hfiles) {
+ FileStatus[] hfileStatuses = fs.listStatus(familyDir);
+ for (FileStatus hfileStatus : hfileStatuses) {
+ long length = hfileStatus.getLen();
+ Path hfile = hfileStatus.getPath();
// Skip "_", reference, HFileLink
String fileName = hfile.getName();
if (fileName.startsWith("_")) continue;
@@ -217,6 +219,11 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LOG.warn("Skipping HFileLink " + fileName);
continue;
}
+ if(length > getConf().getLong(HConstants.HREGION_MAX_FILESIZE,
+ HConstants.DEFAULT_MAX_FILE_SIZE)) {
+ LOG.warn("Trying to bulk load hfile " + hfofDir.toString() + " with size: " +
+ length + " bytes can be problematic as it may lead to oversplitting.");
+ }
ret.add(new LoadQueueItem(family, hfile));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/241ad0d7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index ad701b7..6a65038 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -709,6 +709,12 @@ public class HStore implements Store {
+ this.getRegionInfo().getRegionNameAsString());
}
+ if(reader.length() > conf.getLong(HConstants.HREGION_MAX_FILESIZE,
+ HConstants.DEFAULT_MAX_FILE_SIZE)) {
+ LOG.warn("Trying to bulk load hfile " + srcPath.toString() + " with size: " +
+ reader.length() + " bytes can be problematic as it may lead to oversplitting.");
+ }
+
if (verifyBulkLoads) {
long verificationStartTime = EnvironmentEdgeManager.currentTime();
LOG.info("Full verification started for bulk load hfile: " + srcPath.toString());