You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2018/08/02 17:17:38 UTC
[15/50] [abbrv] hadoop git commit: HADOOP-15637.
LocalFs#listLocatedStatus does not filter out hidden .crc files. Contributed
by Erik Krogen.
HADOOP-15637. LocalFs#listLocatedStatus does not filter out hidden .crc files. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8f952ef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8f952ef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8f952ef
Branch: refs/heads/YARN-7402
Commit: e8f952ef06ae05d2b504300d6f19beb8a052b6f1
Parents: 3517a47
Author: Chen Liang <cl...@apache.org>
Authored: Mon Jul 30 10:25:07 2018 -0700
Committer: Chen Liang <cl...@apache.org>
Committed: Mon Jul 30 10:25:07 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/ChecksumFs.java | 37 +++++++++++++++++++
.../fs/FileContextMainOperationsBaseTest.java | 38 ++++++++++++++++++++
2 files changed, 75 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8f952ef/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
index 75622ad..c56f6e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java
@@ -27,10 +27,12 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
+import java.util.NoSuchElementException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Options.ChecksumOpt;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
@@ -527,4 +529,39 @@ public abstract class ChecksumFs extends FilterFs {
}
return results.toArray(new FileStatus[results.size()]);
}
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f)
+ throws AccessControlException, FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ final RemoteIterator<LocatedFileStatus> iter =
+ getMyFs().listLocatedStatus(f);
+ return new RemoteIterator<LocatedFileStatus>() {
+
+ private LocatedFileStatus next = null;
+
+ @Override
+ public boolean hasNext() throws IOException {
+ while (next == null && iter.hasNext()) {
+ LocatedFileStatus unfilteredNext = iter.next();
+ if (!isChecksumFile(unfilteredNext.getPath())) {
+ next = unfilteredNext;
+ }
+ }
+ return next != null;
+ }
+
+ @Override
+ public LocatedFileStatus next() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ LocatedFileStatus tmp = next;
+ next = null;
+ return tmp;
+ }
+
+ };
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8f952ef/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index 62ecd9f..c07a6ff 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -369,6 +369,44 @@ public abstract class FileContextMainOperationsBaseTest {
pathsIterator = fc.listStatus(getTestRootPath(fc, "test/hadoop/a"));
Assert.assertFalse(pathsIterator.hasNext());
}
+
+ @Test
+ public void testListFiles() throws Exception {
+ Path[] testDirs = {
+ getTestRootPath(fc, "test/dir1"),
+ getTestRootPath(fc, "test/dir1/dir1"),
+ getTestRootPath(fc, "test/dir2")
+ };
+ Path[] testFiles = {
+ new Path(testDirs[0], "file1"),
+ new Path(testDirs[0], "file2"),
+ new Path(testDirs[1], "file2"),
+ new Path(testDirs[2], "file1")
+ };
+
+ for (Path path : testDirs) {
+ fc.mkdir(path, FsPermission.getDefault(), true);
+ }
+ for (Path p : testFiles) {
+ FSDataOutputStream out = fc.create(p).build();
+ out.writeByte(0);
+ out.close();
+ }
+
+ RemoteIterator<LocatedFileStatus> filesIterator =
+ fc.util().listFiles(getTestRootPath(fc, "test"), true);
+ LocatedFileStatus[] fileStats =
+ new LocatedFileStatus[testFiles.length];
+ for (int i = 0; i < fileStats.length; i++) {
+ assertTrue(filesIterator.hasNext());
+ fileStats[i] = filesIterator.next();
+ }
+ assertFalse(filesIterator.hasNext());
+
+ for (Path p : testFiles) {
+ assertTrue(containsPath(p, fileStats));
+ }
+ }
@Test
public void testListStatusFilterWithNoMatches() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org