You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2012/11/09 22:29:47 UTC
svn commit: r1407652 - in /hadoop/common/branches/branch-1: CHANGES.txt
src/core/org/apache/hadoop/fs/FileUtil.java
src/test/org/apache/hadoop/fs/TestFileUtil.java
src/test/org/apache/hadoop/hdfs/TestDFSShell.java
Author: suresh
Date: Fri Nov 9 21:29:47 2012
New Revision: 1407652
URL: http://svn.apache.org/viewvc?rev=1407652&view=rev
Log:
HADOOP-8963. CopyFromLocal doesn't always create user directory. Contributed by Arpit Gupta.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileUtil.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDFSShell.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1407652&r1=1407651&r2=1407652&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Fri Nov 9 21:29:47 2012
@@ -300,6 +300,9 @@ Release 1.2.0 - unreleased
HDFS-4168. Fix a NullPointerException in FSNamesystem.removeBlocks(..).
(Jing Zhao via szetszwo)
+ HADOOP-8963. CopyFromLocal doesn't always create user directory.
+ (Arpit Gupta via suresh)
+
Release 1.1.1 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileUtil.java?rev=1407652&r1=1407651&r2=1407652&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/FileUtil.java Fri Nov 9 21:29:47 2012
@@ -362,6 +362,8 @@ public class FileUtil {
} else if (!overwrite) {
throw new IOException("Target " + dst + " already exists");
}
+ } else if (dst.toString().isEmpty()) {
+ return checkDest(null, dstFS, new Path(srcName), overwrite);
}
return dst;
}
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileUtil.java?rev=1407652&r1=1407651&r2=1407652&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/fs/TestFileUtil.java Fri Nov 9 21:29:47 2012
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.fs;
+import static org.junit.Assert.assertTrue;
+
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
@@ -25,6 +27,9 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -319,4 +324,51 @@ public class TestFileUtil {
long du = FileUtil.getDU(TEST_DIR);
Assert.assertEquals(du, 0);
}
+
+ /**
+ * Test that copies file from local filesystem to hdfs. It tests the overwrite
+ * flag is respected when copying files over.
+ */
+ @Test
+ public void testCopy() throws IOException {
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
+ DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
+ // local file system
+ LocalFileSystem lfs = FileSystem.getLocal(conf);
+ try {
+ String fname = "testCopyFromLocal1.txt";
+ File f = new File(TEST_DIR, fname);
+ f.createNewFile();
+ // copy the file to hdfs with overwrite = false
+ boolean status = FileUtil.copy(lfs, new Path(f.getAbsolutePath()), dfs,
+ new Path("."), false, true, conf);
+ assertTrue(status);
+ // make sure the file exists on hdfs
+ assertTrue("File " + fname + " does not exist on hdfs.",
+ dfs.exists(new Path(fname)));
+ // copy the same file to hdfs with overwrite and it should pass
+ status = FileUtil.copy(lfs, new Path(f.getAbsolutePath()), dfs, new Path(
+ "."), false, true, conf);
+ assertTrue(status);
+
+ try {
+ // copy the same file to hdfs with overwrite as false and it should
+ // thrown an exception, catch it and make sure the file name is part of
+ // the message
+ status = FileUtil.copy(lfs, new Path(f.getAbsolutePath()), dfs,
+ new Path("."), false, false, conf);
+ } catch (IOException expected) {
+ assertTrue("Exception message doesn't contain filename " + fname,
+ expected.getMessage().indexOf(fname) >= 0);
+ }
+ } finally {
+ try {
+ dfs.close();
+ lfs.close();
+ } catch (Exception e) {
+ }
+ cluster.shutdown();
+ }
+ }
}
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1407652&r1=1407651&r2=1407652&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDFSShell.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDFSShell.java Fri Nov 9 21:29:47 2012
@@ -1270,6 +1270,108 @@ public class TestDFSShell extends TestCa
cluster.shutdown();
}
}
+
+ /**
+ * run copyFromLocal when home dir does not exist and use . as the destination
+ */
+ public void testCopyFromLocal1() throws IOException {
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
+ DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
+ FsShell shell = new FsShell();
+ shell.setConf(conf);
+ try {
+ String fname = "testCopyFromLocal1.txt";
+ File f = createLocalFile(new File(TEST_ROOT_DIR, fname));
+
+ // copy the file to hdfs when home dir does not exist
+ int exitCode = runCmd(shell, "-copyFromLocal", f.getAbsolutePath(), ".");
+
+ assertEquals("cmd exit code is not 0", 0, exitCode);
+
+ // make sure the file exists on hdfs
+ assertTrue("File " + fname + " does not exist on hdfs.",
+ dfs.exists(new Path(fname)));
+
+ // copy the file to hdfs again and it should fail
+ exitCode = runCmd(shell, "-copyFromLocal", f.getAbsolutePath(), ".");
+ assertTrue("cmd exit code is 0 when it should not be -> " + exitCode,
+ exitCode != 0);
+
+ // copy another file onto hdfs when the home dir exists and make sure it
+ // works
+ String fname2 = "testCopyFromLocal1.2.txt";
+ File f2 = createLocalFile(new File(TEST_ROOT_DIR, fname2));
+
+ // copy the file to hdfs
+ exitCode = runCmd(shell, "-copyFromLocal", f2.getAbsolutePath(), ".");
+
+ assertEquals("cmd exit code is not 0", 0, exitCode);
+
+ // make sure the file exists on hdfs
+ assertTrue("File " + fname2 + " does not exist on hdfs.",
+ dfs.exists(new Path(fname2)));
+
+ // copy another file onto hdfs when the home dir exists and make sure it
+ // works when we use file name as the destination
+ String fname3 = "testCopyFromLocal1.3.txt";
+ File f3 = createLocalFile(new File(TEST_ROOT_DIR, fname3));
+
+ // copy the file to hdfs
+ exitCode = runCmd(shell, "-copyFromLocal", f3.getAbsolutePath(), fname3);
+
+ assertEquals("cmd exit code is not 0", 0, exitCode);
+
+ // make sure the file exists on hdfs
+ assertTrue("File " + fname3 + " does not exist on hdfs.",
+ dfs.exists(new Path(fname3)));
+ } finally {
+ try {
+ dfs.close();
+ } catch (Exception e) {
+ }
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * run copyFromLocal when home dir does not exist and use the file name as the
+ * destination
+ */
+ public void testCopyFromLocal2() throws IOException {
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
+ DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
+ FsShell shell = new FsShell();
+ shell.setConf(conf);
+ try {
+ String fname = "testCopyFromLocal2.txt";
+ File f = createLocalFile(new File(TEST_ROOT_DIR, fname));
+
+ // copy the file to hdfs
+ int exitCode = runCmd(shell, "-copyFromLocal", f.getAbsolutePath(), fname);
+
+ assertEquals("cmd exit code is not 0", 0, exitCode);
+
+ // make sure the file exists on hdfs
+ assertTrue("File " + fname + " does not exist on hdfs.",
+ dfs.exists(new Path(fname)));
+
+ // copy the file to hdfs again and make sure it fails.
+ exitCode = runCmd(shell, "-copyFromLocal", f.getAbsolutePath(), fname);
+
+ assertTrue("cmd exit code is 0 when it should not be -> " + exitCode,
+ exitCode != 0);
+
+ } finally {
+ try {
+ dfs.close();
+ } catch (Exception e) {
+ }
+ cluster.shutdown();
+ }
+ }
+
private static String runLsr(final FsShell shell, String root, int returnvalue
) throws Exception {
System.out.println("root=" + root + ", returnvalue=" + returnvalue);