You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crunch.apache.org by jw...@apache.org on 2012/12/13 20:16:25 UTC
git commit: CRUNCH-131: Use globStatus in SourceTargetHelper so that
wildcard patterns work properly with Crunch. Contributed by Dave Beech.
Updated Branches:
refs/heads/master 092ef5d01 -> 4a3aa0dfd
CRUNCH-131: Use globStatus in SourceTargetHelper so that wildcard patterns work properly with Crunch. Contributed by Dave Beech.
Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/4a3aa0df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/4a3aa0df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/4a3aa0df
Branch: refs/heads/master
Commit: 4a3aa0dfd9dbdfd2532b03776d87b99f71fae5a5
Parents: 092ef5d
Author: Josh Wills <jw...@apache.org>
Authored: Thu Dec 13 11:15:50 2012 -0800
Committer: Josh Wills <jw...@apache.org>
Committed: Thu Dec 13 11:15:50 2012 -0800
----------------------------------------------------------------------
.../org/apache/crunch/io/SourceTargetHelper.java | 22 ++-------------
1 files changed, 3 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/4a3aa0df/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java b/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
index bbe5eaa..65fb35a 100644
--- a/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
+++ b/crunch/src/main/java/org/apache/crunch/io/SourceTargetHelper.java
@@ -17,7 +17,6 @@
*/
package org.apache.crunch.io;
-import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.commons.logging.Log;
@@ -40,28 +39,13 @@ public class SourceTargetHelper {
}
public static long getPathSize(FileSystem fs, Path path) throws IOException {
-
- if (!fs.exists(path)) {
+ FileStatus[] stati = fs.globStatus(path);
+ if (stati == null || stati.length == 0) {
return -1L;
}
-
- FileStatus[] stati = null;
- try {
- stati = fs.listStatus(path);
- if (stati == null) {
- throw new FileNotFoundException(path + " doesn't exist");
- }
- } catch (FileNotFoundException e) {
- LOG.warn("Returning 0 for getPathSize on non-existant path '" + path + "'");
- return 0L;
- }
-
- if (stati.length == 0) {
- return 0L;
- }
long size = 0;
for (FileStatus status : stati) {
- size += status.getLen();
+ size += fs.getContentSummary(status.getPath()).getLength();
}
return size;
}