You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sa...@apache.org on 2013/12/31 23:57:25 UTC
svn commit: r1554589 - in /hadoop/common/branches/branch-1: CHANGES.txt
src/test/org/apache/hadoop/fs/TestCopyFiles.java
src/tools/org/apache/hadoop/tools/DistCp.java
Author: sandy
Date: Tue Dec 31 22:57:25 2013
New Revision: 1554589
URL: http://svn.apache.org/r1554589
Log:
MAPREDUCE-5698. Backport MAPREDUCE-1285 to branch-1 (Yongjun Zhang via Sandy Ryza)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestCopyFiles.java
hadoop/common/branches/branch-1/src/tools/org/apache/hadoop/tools/DistCp.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1554589&r1=1554588&r2=1554589&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Dec 31 22:57:25 2013
@@ -175,6 +175,9 @@ Release 1.3.0 - unreleased
MAPREDUCE-5702. TaskLogServlet.printTaskLog() has spurious HTML closing
tags (rkanter via kasha)
+ MAPREDUCE-5698. Backport MAPREDUCE-1285 to branch-1 (Yongjun Zhang via
+ Sandy Ryza)
+
Release 1.2.2 - unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestCopyFiles.java?rev=1554589&r1=1554588&r2=1554589&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestCopyFiles.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestCopyFiles.java Tue Dec 31 22:57:25 2013
@@ -909,6 +909,40 @@ public class TestCopyFiles extends TestC
}
}
+ /**
+ * verify that -delete option works for other {@link FileSystem}
+ * implementations. See MAPREDUCE-1285
+ */
+ public void testDeleteLocal() throws Exception {
+ MiniDFSCluster cluster = null;
+ try {
+ Configuration conf = new Configuration();
+ final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
+ cluster = new MiniDFSCluster(conf, 1, true, null);
+ final FileSystem hdfs = cluster.getFileSystem();
+ final String namenode = FileSystem.getDefaultUri(conf).toString();
+ if (namenode.startsWith("hdfs://")) {
+ MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
+ String destdir = TEST_ROOT_DIR + "/destdat";
+ MyFile[] localFiles = createFiles(localfs, destdir);
+ ToolRunner.run(new DistCp(conf), new String[] { "-delete", "-update",
+ "-log", "/logs", namenode + "/srcdat",
+ "file:///" + TEST_ROOT_DIR + "/destdat" });
+ assertTrue("Source and destination directories do not match.",
+ checkFiles(localfs, destdir, files));
+ assertTrue("Log directory does not exist.",
+ hdfs.exists(new Path("/logs")));
+ deldir(localfs, destdir);
+ deldir(hdfs, "/logs");
+ deldir(hdfs, "/srcdat");
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
static void create(FileSystem fs, Path f) throws IOException {
FSDataOutputStream out = fs.create(f);
try {
Modified: hadoop/common/branches/branch-1/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/tools/org/apache/hadoop/tools/DistCp.java?rev=1554589&r1=1554588&r2=1554589&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/common/branches/branch-1/src/tools/org/apache/hadoop/tools/DistCp.java Tue Dec 31 22:57:25 2013
@@ -1275,7 +1275,7 @@ public class DistCp implements Tool {
//write dst lsr results
final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr");
final SequenceFile.Writer writer = SequenceFile.createWriter(jobfs, jobconf,
- dstlsr, Text.class, FileStatus.class,
+ dstlsr, Text.class, dstroot.getClass(),
SequenceFile.CompressionType.NONE);
try {
//do lsr to get all file statuses in dstroot