You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2014/06/02 03:49:44 UTC
git commit: CRUNCH-408: Make HFileSource correctly estimate file
sizes when there are wildcards in the path. Contributed by Chao Shi.
Repository: crunch
Updated Branches:
refs/heads/master c135bba63 -> fcb861edc
CRUNCH-408: Make HFileSource correctly estimate file sizes when there are wildcards in the path. Contributed by Chao Shi.
Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/fcb861ed
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/fcb861ed
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/fcb861ed
Branch: refs/heads/master
Commit: fcb861edce7a5b2a76e2a21300b054932174bc47
Parents: c135bba
Author: Josh Wills <jw...@apache.org>
Authored: Sun Jun 1 13:29:46 2014 -0700
Committer: Josh Wills <jw...@apache.org>
Committed: Sun Jun 1 13:29:46 2014 -0700
----------------------------------------------------------------------
.../org/apache/crunch/io/hbase/HFileSource.java | 19 +------------------
1 file changed, 1 insertion(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/crunch/blob/fcb861ed/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
----------------------------------------------------------------------
diff --git a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
index b8b6df2..c21cc47 100644
--- a/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
+++ b/crunch-hbase/src/main/java/org/apache/crunch/io/hbase/HFileSource.java
@@ -120,28 +120,11 @@ public class HFileSource extends FileSourceImpl<KeyValue> implements ReadableSou
long sum = 0;
for (Path path : getPaths()) {
try {
- sum += getSizeInternal(conf, path);
+ sum += SourceTargetHelper.getPathSize(conf, new Path(path, "*"));
} catch (IOException e) {
LOG.warn("Failed to estimate size of " + path);
}
}
return sum;
}
-
- private long getSizeInternal(Configuration conf, Path path) throws IOException {
- FileSystem fs = path.getFileSystem(conf);
- FileStatus[] statuses = fs.listStatus(path, HFileInputFormat.HIDDEN_FILE_FILTER);
- if (statuses == null) {
- return 0;
- }
- long sum = 0;
- for (FileStatus status : statuses) {
- if (status.isDir()) {
- sum += SourceTargetHelper.getPathSize(fs, status.getPath());
- } else {
- sum += status.getLen();
- }
- }
- return sum;
- }
}