You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/03/17 23:10:19 UTC
svn commit: r638094 - in /hadoop/core/trunk: CHANGES.txt
src/java/org/apache/hadoop/util/CopyFiles.java
src/test/org/apache/hadoop/fs/TestCopyFiles.java
Author: cdouglas
Date: Mon Mar 17 15:10:16 2008
New Revision: 638094
URL: http://svn.apache.org/viewvc?rev=638094&view=rev
Log:
HADOOP-3011. Prohibit distcp from overwriting directories on the
destination filesystem with files.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java
hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=638094&r1=638093&r2=638094&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Mar 17 15:10:16 2008
@@ -263,6 +263,13 @@
HADOOP-3006. Fix wrong packet size reported by DataNode when a block
is being replicated. (rangadi)
+Release 0.16.2 - Unreleased
+
+ BUG FIXES
+
+ HADOOP-3011. Prohibit distcp from overwriting directories on the
+ destination filesystem with files. (cdouglas)
+
Release 0.16.1 - 2008-03-13
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java?rev=638094&r1=638093&r2=638094&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/util/CopyFiles.java Mon Mar 17 15:10:16 2008
@@ -333,8 +333,16 @@
else {
if (totfiles == 1) {
// Copying a single file; use dst path provided by user as destination
- // rather than destination directory
- absdst = absdst.getParent();
+ // rather than destination directory, if a file
+ Path dstparent = absdst.getParent();
+ if (!(destFileSys.exists(dstparent) &&
+ destFileSys.getFileStatus(dstparent).isDir())) {
+ absdst = dstparent;
+ }
+ }
+ if (destFileSys.exists(absdst) &&
+ destFileSys.getFileStatus(absdst).isDir()) {
+ throw new IOException(absdst + " is a directory");
}
rename(destFileSys, tmpfile, absdst);
}
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java?rev=638094&r1=638093&r2=638094&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/fs/TestCopyFiles.java Mon Mar 17 15:10:16 2008
@@ -57,7 +57,9 @@
private long seed = 0L;
MyFile() {
- int nLevels = gen.nextInt(MAX_LEVELS);
+ this(gen.nextInt(MAX_LEVELS));
+ }
+ MyFile(int nLevels) {
String xname = "";
if (nLevels != 0) {
int[] levels = new int[nLevels];
@@ -101,8 +103,9 @@
return files;
}
- static MyFile createFile(Path root, FileSystem fs) throws IOException {
- MyFile f = new MyFile();
+ static MyFile createFile(Path root, FileSystem fs, int levels)
+ throws IOException {
+ MyFile f = levels < 0 ? new MyFile() : new MyFile(levels);
Path p = new Path(root, f.getName());
FSDataOutputStream out = fs.create(p);
byte[] toWrite = new byte[f.getSize()];
@@ -113,6 +116,10 @@
return f;
}
+ static MyFile createFile(Path root, FileSystem fs) throws IOException {
+ return createFile(root, fs, -1);
+ }
+
/** check if the files have been copied correctly. */
private static boolean checkFiles(String fsname, String topdir, MyFile[] files)
throws IOException {
@@ -413,6 +420,25 @@
"file:///"+TEST_ROOT_DIR+"/dest2/"+fname});
assertTrue("Source and destination directories do not match.",
checkFiles("local", TEST_ROOT_DIR+"/dest2", files));
+ //copy single file to existing dir
+ deldir("local", TEST_ROOT_DIR+"/dest2");
+ fs.mkdirs(new Path(TEST_ROOT_DIR+"/dest2"));
+ MyFile[] files2 = {createFile(root, fs, 0)};
+ String sname = files2[0].getName();
+ ToolRunner.run(new CopyFiles(new Configuration()),
+ new String[] {"-update",
+ "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
+ "file:///"+TEST_ROOT_DIR+"/dest2/"});
+ assertTrue("Source and destination directories do not match.",
+ checkFiles("local", TEST_ROOT_DIR+"/dest2", files2));
+ updateFiles("local", TEST_ROOT_DIR+"/srcdat", files2, 1);
+ //copy single file to existing dir w/ dst name conflict
+ ToolRunner.run(new CopyFiles(new Configuration()),
+ new String[] {"-update",
+ "file:///"+TEST_ROOT_DIR+"/srcdat/"+sname,
+ "file:///"+TEST_ROOT_DIR+"/dest2/"});
+ assertTrue("Source and destination directories do not match.",
+ checkFiles("local", TEST_ROOT_DIR+"/dest2", files2));
}
finally {
deldir("local", TEST_ROOT_DIR+"/destdat");