You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ma...@apache.org on 2010/08/20 05:46:08 UTC
svn commit: r987374 - in /hadoop/common/trunk: CHANGES.txt
src/java/org/apache/hadoop/fs/ChecksumFs.java
src/java/org/apache/hadoop/fs/FileContext.java
src/test/core/org/apache/hadoop/fs/FileContextUtilBase.java
Author: mahadev
Date: Fri Aug 20 03:46:07 2010
New Revision: 987374
URL: http://svn.apache.org/viewvc?rev=987374&view=rev
Log:
HADOOP-6906. FileContext copy() utility doesn't work with recursive copying of directories. (vinod k v via mahadev)
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextUtilBase.java
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=987374&r1=987373&r2=987374&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Fri Aug 20 03:46:07 2010
@@ -205,6 +205,9 @@ Trunk (unreleased changes)
HADOOP-6482. GenericOptionsParser constructor that takes Options and
String[] ignores options. (Eli Collins via jghoman)
+ HADOOP-6906. FileContext copy() utility doesn't work with recursive
+ copying of directories. (vinod k v via mahadev)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java?rev=987374&r1=987373&r2=987374&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/ChecksumFs.java Fri Aug 20 03:46:07 2010
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
import java.io.*;
import java.net.URISyntaxException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
@@ -478,4 +479,19 @@ public abstract class ChecksumFs extends
long inPos, FSDataInputStream sums, long sumsPos) {
return false;
}
+
+ @Override
+ protected FileStatus[] listStatus(Path f) throws IOException,
+ UnresolvedLinkException {
+ ArrayList<FileStatus> results = new ArrayList<FileStatus>();
+ FileStatus[] listing = getMyFs().listStatus(f);
+ if (listing != null) {
+ for (int i = 0; i < listing.length; i++) {
+ if (!isChecksumFile(listing[i].getPath())) {
+ results.add(listing[i]);
+ }
+ }
+ }
+ return results.toArray(new FileStatus[results.size()]);
+ }
}
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java?rev=987374&r1=987373&r2=987374&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FileContext.java Fri Aug 20 03:46:07 2010
@@ -2017,8 +2017,8 @@ public final class FileContext {
mkdir(qDst, FsPermission.getDefault(), true);
FileStatus[] contents = listStatus(qSrc);
for (FileStatus content : contents) {
- copy(content.getPath(), new Path(qDst, content.getPath()),
- deleteSource, overwrite);
+ copy(makeQualified(content.getPath()), makeQualified(new Path(qDst,
+ content.getPath().getName())), deleteSource, overwrite);
}
} else {
InputStream in=null;
@@ -2062,7 +2062,8 @@ public final class FileContext {
// Recurse to check if dst/srcName exists.
checkDest(null, new Path(dst, srcName), overwrite);
} else if (!overwrite) {
- throw new IOException("Target " + dst + " already exists");
+ throw new IOException("Target " + new Path(dst, srcName)
+ + " already exists");
}
} catch (FileNotFoundException e) {
// dst does not exist - OK to copy.
@@ -2098,8 +2099,9 @@ public final class FileContext {
private static boolean isSameFS(Path qualPath1, Path qualPath2) {
URI srcUri = qualPath1.toUri();
URI dstUri = qualPath2.toUri();
- return (srcUri.getAuthority().equals(dstUri.getAuthority()) && srcUri
- .getAuthority().equals(dstUri.getAuthority()));
+ return (srcUri.getScheme().equals(dstUri.getScheme()) &&
+ !(srcUri.getAuthority() != null && dstUri.getAuthority() != null && srcUri
+ .getAuthority().equals(dstUri.getAuthority())));
}
/**
@@ -2176,7 +2178,7 @@ public final class FileContext {
// NB: More than one AbstractFileSystem can match a scheme, eg
// "file" resolves to LocalFs but could have come by RawLocalFs.
AbstractFileSystem fs = fc.getFSofPath(p);
-
+
// Loop until all symlinks are resolved or the limit is reached
for (boolean isLink = true; isLink;) {
try {
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextUtilBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextUtilBase.java?rev=987374&r1=987373&r2=987374&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextUtilBase.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/fs/FileContextUtilBase.java Fri Aug 20 03:46:07 2010
@@ -17,15 +17,17 @@
*/
package org.apache.hadoop.fs;
+import static org.apache.hadoop.fs.FileContextTestHelper.getTestRootPath;
+import static org.apache.hadoop.fs.FileContextTestHelper.readFile;
+import static org.apache.hadoop.fs.FileContextTestHelper.writeFile;
+import static org.junit.Assert.assertTrue;
+
import java.util.Arrays;
import org.apache.hadoop.util.StringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import static org.junit.Assert.*;
-
-import static org.apache.hadoop.fs.FileContextTestHelper.*;
/**
* <p>
@@ -80,4 +82,27 @@ public abstract class FileContextUtilBas
assertTrue("Copied files does not match ",Arrays.equals(ts.getBytes(),
readFile(fc,file2,ts.getBytes().length)));
}
-}
\ No newline at end of file
+
+ @Test
+ public void testRecursiveFcCopy() throws Exception {
+
+ final String ts = "some random text";
+ Path dir1 = getTestRootPath(fc, "dir1");
+ Path dir2 = getTestRootPath(fc, "dir2");
+
+ Path file1 = new Path(dir1, "file1");
+ fc.mkdir(dir1, null, false);
+ writeFile(fc, file1, ts.getBytes());
+ assertTrue(fc.util().exists(file1));
+
+ Path file2 = new Path(dir2, "file1");
+
+ fc.util().copy(dir1, dir2);
+
+ // verify that newly copied file2 exists
+ assertTrue("Failed to copy file2 ", fc.util().exists(file2));
+ // verify that file2 contains test string
+ assertTrue("Copied files does not match ",Arrays.equals(ts.getBytes(),
+ readFile(fc,file2,ts.getBytes().length)));
+ }
+}