You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2014/06/02 05:42:21 UTC
git commit: CRUNCH-408: Fix the HFileSource globStatus check to work
on Hadoop2
Repository: crunch
Updated Branches:
refs/heads/master f2dee62db -> fcb1aa415
CRUNCH-408: Fix the HFileSource globStatus check to work on Hadoop2
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/fcb1aa41
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/fcb1aa41
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/fcb1aa41
Branch: refs/heads/master
Commit: fcb1aa4158f90001c114daa90d68ce99fd8870e4
Parents: f2dee62
Author: Josh Wills <jw...@apache.org>
Authored: Sun Jun 1 20:41:01 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Sun Jun 1 20:41:01 2014 -0700
----------------------------------------------------------------------
.../org/apache/crunch/io/hbase/HFileSource.java | 20 +++++++++++++++++++-
1 file changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/fcb1aa41/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
index c21cc47..47abe9a 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
@@ -120,10 +120,28 @@ public class HFileSource extends FileSourceImpl<KeyValue> implements ReadableSou
long sum = 0;
for (Path path : getPaths()) {
try {
- sum += SourceTargetHelper.getPathSize(conf, new Path(path, "*"));
+ sum += getSizeInternal(conf, path);
} catch (IOException e) {
LOG.warn("Failed to estimate size of " + path);
}
+ System.out.println("Size after read of path = " + path.toString() + " = " + sum);
+ }
+ return sum;
+ }
+
+ private long getSizeInternal(Configuration conf, Path path) throws IOException {
+ FileSystem fs = path.getFileSystem(conf);
+ FileStatus[] statuses = fs.globStatus(path, HFileInputFormat.HIDDEN_FILE_FILTER);
+ if (statuses == null) {
+ return 0;
+ }
+ long sum = 0;
+ for (FileStatus status : statuses) {
+ if (status.isDir()) {
+ sum += SourceTargetHelper.getPathSize(fs, status.getPath());
+ } else {
+ sum += status.getLen();
+ }
}
return sum;
}