You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2015/03/03 03:27:54 UTC
sqoop git commit: SQOOP-2156: Sqoop2:
HdfsUtils.getOutputMapreduceFiles should ignore hidden files
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 c408ac5a4 -> 934a00976
SQOOP-2156: Sqoop2: HdfsUtils.getOutputMapreduceFiles should ignore hidden files
(Jarek Jarcec Cecho via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/934a0097
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/934a0097
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/934a0097
Branch: refs/heads/sqoop2
Commit: 934a009762719a2459aa8f85e0c300645c5b2d20
Parents: c408ac5
Author: Abraham Elmahrek <ab...@apache.org>
Authored: Mon Mar 2 18:27:28 2015 -0800
Committer: Abraham Elmahrek <ab...@apache.org>
Committed: Mon Mar 2 18:27:28 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/sqoop/test/utils/HdfsUtils.java | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/934a0097/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
index 5a55b8b..79700a6 100644
--- a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
+++ b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
@@ -20,6 +20,7 @@ package org.apache.sqoop.test.utils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.log4j.Logger;
import java.io.BufferedWriter;
@@ -37,6 +38,14 @@ public class HdfsUtils {
private static final char PATH_SEPARATOR = '/';
+ public static PathFilter filterHiddenFiles = new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ String fileName = path.getName();
+ return !fileName.startsWith("_") && !fileName.startsWith(".");
+ }
+ };
+
/**
* Get list of mapreduce output files from given directory.
*
@@ -45,9 +54,9 @@ public class HdfsUtils {
* @throws IOException
* @throws FileNotFoundException
*/
- public static Path [] getOutputMapreduceFiles(FileSystem fs, String directory) throws FileNotFoundException, IOException {
+ public static Path [] getOutputMapreduceFiles(FileSystem fs, String directory) throws IOException {
LinkedList<Path> files = new LinkedList<Path>();
- for (FileStatus fileStatus : fs.listStatus(new Path(directory))) {
+ for (FileStatus fileStatus : fs.listStatus(new Path(directory), filterHiddenFiles)) {
LOG.debug("Found mapreduce output file: " + fileStatus.getPath() + " with size " + fileStatus.getLen());
files.add(fileStatus.getPath());
}