You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2015/03/13 18:23:06 UTC
[1/4] hbase git commit: HBASE-13227 LoadIncrementalHFile should skip
non-files inside a possible family-dir
Repository: hbase
Updated Branches:
refs/heads/0.98 41f865378 -> 2f831915d
refs/heads/branch-1 aa93b176f -> 7229a0a47
refs/heads/branch-1.0 7a508a39a -> 3c8bb9424
refs/heads/master 0f69ea409 -> d2d4ef14f
HBASE-13227 LoadIncrementalHFile should skip non-files inside a possible family-dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7229a0a4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7229a0a4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7229a0a4
Branch: refs/heads/branch-1
Commit: 7229a0a47df97fdc418308be17d2d91f2d534fcd
Parents: aa93b17
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Fri Mar 13 17:11:27 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Fri Mar 13 17:12:11 2015 +0000
----------------------------------------------------------------------
.../hbase/mapreduce/LoadIncrementalHFiles.java | 16 ++++++++++++++--
.../hbase/mapreduce/TestLoadIncrementalHFiles.java | 6 ++++--
2 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7229a0a4/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 9c120c2..53efc42 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -209,6 +209,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
byte[] family = familyDir.getName().getBytes();
FileStatus[] hfileStatuses = fs.listStatus(familyDir);
for (FileStatus hfileStatus : hfileStatuses) {
+ if (!hfileStatus.isFile()) {
+ LOG.warn("Skipping non-file " + hfileStatus);
+ continue;
+ }
long length = hfileStatus.getLen();
Path hfile = hfileStatus.getPath();
// Skip "_", reference, HFileLink
@@ -293,10 +297,18 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LoadQueueItem lqi = queueIter.next();
String familyNameInHFile = Bytes.toString(lqi.family);
if (!familyNames.contains(familyNameInHFile)) {
- if (HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath)) {
+ boolean isValid = false;
+ try {
+ isValid = HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath);
+ if (!isValid) {
+ LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
+ }
+ } catch (FileNotFoundException e) {
+ LOG.warn("the file " + lqi + " was removed");
+ }
+ if (isValid) {
unmatchedFamilies.add(familyNameInHFile);
} else {
- LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
queueIter.remove();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7229a0a4/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 3d05782..7cc57f5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -308,8 +308,9 @@ public class TestLoadIncrementalHFiles {
}
/**
- * Write a random data file in a dir with a valid family name but not part of the table families
- * we should we able to bulkload without getting the unmatched family exception. HBASE-13037
+ * Write a random data file and a non-file in a dir with a valid family name
+ * but not part of the table families. we should we able to bulkload without
+ * getting the unmatched family exception. HBASE-13037/HBASE-13227
*/
@Test(timeout = 60000)
public void testNonHfileFolderWithUnmatchedFamilyName() throws Exception {
@@ -324,6 +325,7 @@ public class TestLoadIncrementalHFiles {
final String NON_FAMILY_FOLDER = "_logs";
Path nonFamilyDir = new Path(dir, NON_FAMILY_FOLDER);
fs.mkdirs(nonFamilyDir);
+ fs.mkdirs(new Path(nonFamilyDir, "non-file"));
createRandomDataFile(fs, new Path(nonFamilyDir, "012356789"), 16 * 1024);
Table table = null;
[4/4] hbase git commit: HBASE-13227 LoadIncrementalHFile should skip
non-files inside a possible family-dir
Posted by mb...@apache.org.
HBASE-13227 LoadIncrementalHFile should skip non-files inside a possible family-dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d2d4ef14
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d2d4ef14
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d2d4ef14
Branch: refs/heads/master
Commit: d2d4ef14f43d48bb37fc53b5a5ad02265a2b1ac7
Parents: 0f69ea4
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Fri Mar 13 17:11:27 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Fri Mar 13 17:22:54 2015 +0000
----------------------------------------------------------------------
.../hbase/mapreduce/LoadIncrementalHFiles.java | 16 ++++++++++++++--
.../hbase/mapreduce/TestLoadIncrementalHFiles.java | 6 ++++--
2 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d2d4ef14/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 0bed818..0f9ade3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -209,6 +209,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
byte[] family = familyDir.getName().getBytes();
FileStatus[] hfileStatuses = fs.listStatus(familyDir);
for (FileStatus hfileStatus : hfileStatuses) {
+ if (!hfileStatus.isFile()) {
+ LOG.warn("Skipping non-file " + hfileStatus);
+ continue;
+ }
long length = hfileStatus.getLen();
Path hfile = hfileStatus.getPath();
// Skip "_", reference, HFileLink
@@ -293,10 +297,18 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LoadQueueItem lqi = queueIter.next();
String familyNameInHFile = Bytes.toString(lqi.family);
if (!familyNames.contains(familyNameInHFile)) {
- if (HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath)) {
+ boolean isValid = false;
+ try {
+ isValid = HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath);
+ if (!isValid) {
+ LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
+ }
+ } catch (FileNotFoundException e) {
+ LOG.warn("the file " + lqi + " was removed");
+ }
+ if (isValid) {
unmatchedFamilies.add(familyNameInHFile);
} else {
- LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
queueIter.remove();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d2d4ef14/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 0a90fd5..e356049 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -308,8 +308,9 @@ public class TestLoadIncrementalHFiles {
}
/**
- * Write a random data file in a dir with a valid family name but not part of the table families
- * we should we able to bulkload without getting the unmatched family exception. HBASE-13037
+ * Write a random data file and a non-file in a dir with a valid family name
+ * but not part of the table families. we should we able to bulkload without
+ * getting the unmatched family exception. HBASE-13037/HBASE-13227
*/
@Test(timeout = 60000)
public void testNonHfileFolderWithUnmatchedFamilyName() throws Exception {
@@ -324,6 +325,7 @@ public class TestLoadIncrementalHFiles {
final String NON_FAMILY_FOLDER = "_logs";
Path nonFamilyDir = new Path(dir, NON_FAMILY_FOLDER);
fs.mkdirs(nonFamilyDir);
+ fs.mkdirs(new Path(nonFamilyDir, "non-file"));
createRandomDataFile(fs, new Path(nonFamilyDir, "012356789"), 16 * 1024);
Table table = null;
[2/4] hbase git commit: HBASE-13227 LoadIncrementalHFile should skip
non-files inside a possible family-dir
Posted by mb...@apache.org.
HBASE-13227 LoadIncrementalHFile should skip non-files inside a possible family-dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3c8bb942
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3c8bb942
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3c8bb942
Branch: refs/heads/branch-1.0
Commit: 3c8bb9424ba9b850aa464c46f8d5a6aa253fba37
Parents: 7a508a3
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Fri Mar 13 17:11:27 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Fri Mar 13 17:17:48 2015 +0000
----------------------------------------------------------------------
.../hbase/mapreduce/LoadIncrementalHFiles.java | 16 ++++++++++++++--
.../hbase/mapreduce/TestLoadIncrementalHFiles.java | 6 ++++--
2 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3c8bb942/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 9beb6d5..b78017d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -209,6 +209,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
byte[] family = familyDir.getName().getBytes();
FileStatus[] hfileStatuses = fs.listStatus(familyDir);
for (FileStatus hfileStatus : hfileStatuses) {
+ if (!hfileStatus.isFile()) {
+ LOG.warn("Skipping non-file " + hfileStatus);
+ continue;
+ }
long length = hfileStatus.getLen();
Path hfile = hfileStatus.getPath();
// Skip "_", reference, HFileLink
@@ -281,10 +285,18 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LoadQueueItem lqi = queueIter.next();
String familyNameInHFile = Bytes.toString(lqi.family);
if (!familyNames.contains(familyNameInHFile)) {
- if (HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath)) {
+ boolean isValid = false;
+ try {
+ isValid = HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath);
+ if (!isValid) {
+ LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
+ }
+ } catch (FileNotFoundException e) {
+ LOG.warn("the file " + lqi + " was removed");
+ }
+ if (isValid) {
unmatchedFamilies.add(familyNameInHFile);
} else {
- LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
queueIter.remove();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3c8bb942/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index abec268..39b6663 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -308,8 +308,9 @@ public class TestLoadIncrementalHFiles {
}
/**
- * Write a random data file in a dir with a valid family name but not part of the table families
- * we should we able to bulkload without getting the unmatched family exception. HBASE-13037
+ * Write a random data file and a non-file in a dir with a valid family name
+ * but not part of the table families. we should we able to bulkload without
+ * getting the unmatched family exception. HBASE-13037/HBASE-13227
*/
@Test(timeout = 60000)
public void testNonHfileFolderWithUnmatchedFamilyName() throws Exception {
@@ -324,6 +325,7 @@ public class TestLoadIncrementalHFiles {
final String NON_FAMILY_FOLDER = "_logs";
Path nonFamilyDir = new Path(dir, NON_FAMILY_FOLDER);
fs.mkdirs(nonFamilyDir);
+ fs.mkdirs(new Path(nonFamilyDir, "non-file"));
createRandomDataFile(fs, new Path(nonFamilyDir, "012356789"), 16 * 1024);
Table table = null;
[3/4] hbase git commit: HBASE-13227 LoadIncrementalHFile should skip
non-files inside a possible family-dir
Posted by mb...@apache.org.
HBASE-13227 LoadIncrementalHFile should skip non-files inside a possible family-dir
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2f831915
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2f831915
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2f831915
Branch: refs/heads/0.98
Commit: 2f831915d86359785395b8bdd6d6ba2698dc2ef0
Parents: 41f8653
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Fri Mar 13 17:11:27 2015 +0000
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Fri Mar 13 17:19:42 2015 +0000
----------------------------------------------------------------------
.../hbase/mapreduce/LoadIncrementalHFiles.java | 16 ++++++++++++++--
.../hbase/mapreduce/TestLoadIncrementalHFiles.java | 6 ++++--
2 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2f831915/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
index 5c71667..a77ea01 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
@@ -207,6 +207,10 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
byte[] family = familyDir.getName().getBytes();
FileStatus[] hfileStatuses = fs.listStatus(familyDir);
for (FileStatus hfileStatus : hfileStatuses) {
+ if (!hfileStatus.isFile()) {
+ LOG.warn("Skipping non-file " + hfileStatus);
+ continue;
+ }
long length = hfileStatus.getLen();
Path hfile = hfileStatus.getPath();
if (hfile.getName().startsWith("_")) continue;
@@ -269,10 +273,18 @@ public class LoadIncrementalHFiles extends Configured implements Tool {
LoadQueueItem lqi = queueIter.next();
String familyNameInHFile = Bytes.toString(lqi.family);
if (!familyNames.contains(familyNameInHFile)) {
- if (HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath)) {
+ boolean isValid = false;
+ try {
+ isValid = HFile.isHFileFormat(lqi.hfilePath.getFileSystem(getConf()), lqi.hfilePath);
+ if (!isValid) {
+ LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
+ }
+ } catch (FileNotFoundException e) {
+ LOG.warn("the file " + lqi + " was removed");
+ }
+ if (isValid) {
unmatchedFamilies.add(familyNameInHFile);
} else {
- LOG.warn("the file " + lqi + " doesn't seems to be an hfile. skipping");
queueIter.remove();
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2f831915/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
index 5a3de64..00fe6d3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestLoadIncrementalHFiles.java
@@ -307,8 +307,9 @@ public class TestLoadIncrementalHFiles {
}
/**
- * Write a random data file in a dir with a valid family name but not part of the table families
- * we should we able to bulkload without getting the unmatched family exception. HBASE-13037
+ * Write a random data file and a non-file in a dir with a valid family name
+ * but not part of the table families. we should we able to bulkload without
+ * getting the unmatched family exception. HBASE-13037/HBASE-13227
*/
@Test(timeout = 60000)
public void testNonHfileFolderWithUnmatchedFamilyName() throws Exception {
@@ -323,6 +324,7 @@ public class TestLoadIncrementalHFiles {
final String NON_FAMILY_FOLDER = "_logs";
Path nonFamilyDir = new Path(dir, NON_FAMILY_FOLDER);
fs.mkdirs(nonFamilyDir);
+ fs.mkdirs(new Path(nonFamilyDir, "non-file"));
createRandomDataFile(fs, new Path(nonFamilyDir, "012356789"), 16 * 1024);
HTable table = null;