You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2009/12/11 02:28:02 UTC
svn commit: r889486 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
src/tools/org/apache/hadoop/tools/DistCp.java
Author: szetszwo
Date: Fri Dec 11 01:28:01 2009
New Revision: 889486
URL: http://svn.apache.org/viewvc?rev=889486&view=rev
Log:
MAPREDUCE-1285. In DistCp.deleteNonexisting(..), get class from the parameter instead of using FileStatus.class. Contributed by Peter Romianowski
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=889486&r1=889485&r2=889486&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Fri Dec 11 01:28:01 2009
@@ -124,6 +124,10 @@
MAPREDUCE-1152. Distinguish between failed and killed tasks in
JobTrackerInstrumentation. (Sharad Agarwal via cdouglas)
+ MAPREDUCE-1285. In DistCp.deleteNonexisting(..), get class from the
+ parameter instead of using FileStatus.class. (Peter Romianowski via
+ szetszwo)
+
Release 0.21.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java?rev=889486&r1=889485&r2=889486&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/tools/TestCopyFiles.java Fri Dec 11 01:28:01 2009
@@ -995,6 +995,41 @@
}
}
+ /**
+ * verify that -delete option works for other {@link FileSystem}
+ * implementations. See MAPREDUCE-1285 */
+ public void testDeleteLocal() throws Exception {
+ MiniDFSCluster cluster = null;
+ try {
+ Configuration conf = new Configuration();
+ final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
+ cluster = new MiniDFSCluster(conf, 1, true, null);
+ final FileSystem hdfs = cluster.getFileSystem();
+ final String namenode = FileSystem.getDefaultUri(conf).toString();
+ if (namenode.startsWith("hdfs://")) {
+ MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
+ String destdir = TEST_ROOT_DIR + "/destdat";
+ MyFile[] localFiles = createFiles(localfs, destdir);
+ ToolRunner.run(new DistCp(conf), new String[] {
+ "-delete",
+ "-update",
+ "-log",
+ "/logs",
+ namenode+"/srcdat",
+ "file:///"+TEST_ROOT_DIR+"/destdat"});
+ assertTrue("Source and destination directories do not match.",
+ checkFiles(localfs, destdir, files));
+ assertTrue("Log directory does not exist.",
+ hdfs.exists(new Path("/logs")));
+ deldir(localfs, destdir);
+ deldir(hdfs, "/logs");
+ deldir(hdfs, "/srcdat");
+ }
+ } finally {
+ if (cluster != null) { cluster.shutdown(); }
+ }
+ }
+
/** test globbing */
public void testGlobbing() throws Exception {
String namenode = null;
@@ -1057,4 +1092,4 @@
}
return results.toString();
}
-}
\ No newline at end of file
+}
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java?rev=889486&r1=889485&r2=889486&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java Fri Dec 11 01:28:01 2009
@@ -1540,7 +1540,7 @@
//write dst lsr results
final Path dstlsr = new Path(jobdir, "_distcp_dst_lsr");
final SequenceFile.Writer writer = SequenceFile.createWriter(jobfs, jobconf,
- dstlsr, Text.class, FileStatus.class,
+ dstlsr, Text.class, dstroot.getClass(),
SequenceFile.CompressionType.NONE);
try {
//do lsr to get all file statuses in dstroot