You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/03/27 02:23:01 UTC
svn commit: r641681 - in /hadoop/core/trunk: ./
src/docs/src/documentation/content/xdocs/ src/java/org/apache/hadoop/fs/
src/test/org/apache/hadoop/dfs/
Author: cdouglas
Date: Wed Mar 26 18:22:59 2008
New Revision: 641681
URL: http://svn.apache.org/viewvc?rev=641681&view=rev
Log:
HADOOP-3091. Modify FsShell command -put to accept multiple sources.
Contributed by Lohit Vijaya Renu.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java
hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Wed Mar 26 18:22:59 2008
@@ -153,6 +153,9 @@
HADOOP-2796. Enables distinguishing exit codes from user code vis-a-vis
HOD's exit code. (Hemanth Yamijala via ddas)
+ HADOOP-3091. Modify FsShell command -put to accept multiple sources.
+ (Lohit Vijaya Renu via cdouglas)
+
OPTIMIZATIONS
HADOOP-2790. Fixed inefficient method hasSpeculativeTask by removing
Modified: hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml (original)
+++ hadoop/core/trunk/src/docs/src/documentation/content/xdocs/hdfs_shell.xml Wed Mar 26 18:22:59 2008
@@ -249,13 +249,16 @@
<section>
<title id="putlink"> put </title>
<p>
- <code>Usage: hadoop dfs -put <localsrc> <dst></code>
+ <code>Usage: hadoop dfs -put <localsrc> ... <dst></code>
</p>
- <p>Copy src from local file system to the destination filesystem. Also reads input from stdin and writes to destination filesystem.<br/>
+ <p>Copy single src, or multiple srcs from local file system to the destination filesystem. Also reads input from stdin and writes to destination filesystem.<br/>
</p>
<ul>
<li>
<code> hadoop dfs -put localfile /user/hadoop/hadoopfile</code>
+ </li>
+ <li>
+ <code> hadoop dfs -put localfile1 localfile2 /user/hadoop/hadoopdir</code>
</li>
<li>
<code> hadoop dfs -put localfile hdfs://host:port/hadoop/hadoopfile</code>
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileSystem.java Wed Mar 26 18:22:59 2008
@@ -1104,6 +1104,15 @@
}
/**
+ * The src files is on the local disk. Add it to FS at
+ * the given dst name, removing the source afterwards.
+ */
+ public void moveFromLocalFile(Path[] srcs, Path dst)
+ throws IOException {
+ copyFromLocalFile(true, true, srcs, dst);
+ }
+
+ /**
* The src file is on the local disk. Add it to FS at
* the given dst name, removing the source afterwards.
*/
@@ -1121,7 +1130,19 @@
throws IOException {
copyFromLocalFile(delSrc, true, src, dst);
}
-
+
+ /**
+ * The src files are on the local disk. Add it to FS at
+ * the given dst name.
+ * delSrc indicates if the source should be removed
+ */
+ public void copyFromLocalFile(boolean delSrc, boolean overwrite,
+ Path[] srcs, Path dst)
+ throws IOException {
+ Configuration conf = getConf();
+ FileUtil.copy(getLocal(conf), srcs, this, dst, delSrc, overwrite, conf);
+ }
+
/**
* The src file is on the local disk. Add it to FS at
* the given dst name.
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FileUtil.java Wed Mar 26 18:22:59 2008
@@ -132,7 +132,46 @@
Configuration conf) throws IOException {
return copy(srcFS, src, dstFS, dst, deleteSource, true, conf);
}
-
+
+ public static boolean copy(FileSystem srcFS, Path[] srcs,
+ FileSystem dstFS, Path dst,
+ boolean deleteSource,
+ boolean overwrite, Configuration conf)
+ throws IOException {
+ boolean gotException = false;
+ boolean returnVal = true;
+ StringBuffer exceptions = new StringBuffer();
+
+ if (srcs.length == 1)
+ return copy(srcFS, srcs[0], dstFS, dst, deleteSource, overwrite, conf);
+
+ // Check if dest is directory
+ if (!dstFS.exists(dst)) {
+ throw new IOException("`" + dst +"': specified destination directory " +
+ "doest not exist");
+ } else {
+ FileStatus sdst = dstFS.getFileStatus(dst);
+ if (!sdst.isDir())
+ throw new IOException("copying multiple files, but last argument `" +
+ dst + "' is not a directory");
+ }
+
+ for (Path src : srcs) {
+ try {
+ if (!copy(srcFS, src, dstFS, dst, deleteSource, overwrite, conf))
+ returnVal = false;
+ } catch (IOException e) {
+ gotException = true;
+ exceptions.append(e.getMessage());
+ exceptions.append("\n");
+ }
+ }
+ if (gotException) {
+ throw new IOException(exceptions.toString());
+ }
+ return returnVal;
+ }
+
/** Copy files between FileSystems. */
public static boolean copy(FileSystem srcFS, Path src,
FileSystem dstFS, Path dst,
@@ -204,7 +243,7 @@
return true;
}
}
-
+
/** Copy local files to a FileSystem. */
public static boolean copy(File src,
FileSystem dstFS, Path dst,
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/fs/FsShell.java Wed Mar 26 18:22:59 2008
@@ -121,26 +121,43 @@
}
}
+
/**
- * Add a local file to the indicated FileSystem name. src is kept.
+ * Add local files to the indicated FileSystem name. src is kept.
*/
- void copyFromLocal(Path src, String dstf) throws IOException {
+ void copyFromLocal(Path[] srcs, String dstf) throws IOException {
Path dstPath = new Path(dstf);
FileSystem dstFs = dstPath.getFileSystem(getConf());
+ dstFs.copyFromLocalFile(false, false, srcs, dstPath);
+ }
+
+ /**
+ * Add a local file to the indicated FileSystem name. src is kept.
+ */
+ void copyFromLocal(Path src, String dstf) throws IOException {
if (src.toString().equals("-")) {
+ Path dstPath = new Path(dstf);
+ FileSystem dstFs = dstPath.getFileSystem(getConf());
copyFromStdin(dstPath, dstFs);
} else {
- dstFs.copyFromLocalFile(false, false, src, new Path(dstf));
+ copyFromLocal(new Path[]{src}, dstf);
}
}
/**
- * Add a local file to the indicated FileSystem name. src is removed.
+ * Add local files to the indicated FileSystem name. src is removed.
*/
- void moveFromLocal(Path src, String dstf) throws IOException {
+ void moveFromLocal(Path[] srcs, String dstf) throws IOException {
Path dstPath = new Path(dstf);
FileSystem dstFs = dstPath.getFileSystem(getConf());
- dstFs.moveFromLocalFile(src, dstPath);
+ dstFs.moveFromLocalFile(srcs, dstPath);
+ }
+
+ /**
+ * Add a local file to the indicated FileSystem name. src is removed.
+ */
+ void moveFromLocal(Path src, String dstf) throws IOException {
+ moveFromLocal((new Path[]{src}), dstf);
}
/**
@@ -1248,8 +1265,9 @@
"hadoop fs [-fs <local | file system URI>] [-conf <configuration file>]\n\t" +
"[-D <property=value>] [-ls <path>] [-lsr <path>] [-du <path>]\n\t" +
"[-dus <path>] [-mv <src> <dst>] [-cp <src> <dst>] [-rm <src>]\n\t" +
- "[-rmr <src>] [-put <localsrc> <dst>] [-copyFromLocal <localsrc> <dst>]\n\t" +
- "[-moveFromLocal <localsrc> <dst>] [" + GET_SHORT_USAGE + "\n\t" +
+ "[-rmr <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" +
+ "[-moveFromLocal <localsrc> ... <dst>] [" +
+ GET_SHORT_USAGE + "\n\t" +
"[-getmerge <src> <localdst> [addnl]] [-cat <src>]\n\t" +
"[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" +
"[-mkdir <path>] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" +
@@ -1318,13 +1336,14 @@
String rmr = "-rmr <src>: \tRemove all directories which match the specified file \n" +
"\t\tpattern. Equivlent to the Unix command \"rm -rf <src>\"\n";
- String put = "-put <localsrc> <dst>: \tCopy a single file from the local file system \n" +
- "\t\tinto fs. \n";
+ String put = "-put <localsrc> ... <dst>: \tCopy files " +
+ "from the local file system \n\t\tinto fs. \n";
- String copyFromLocal = "-copyFromLocal <localsrc> <dst>: Identical to the -put command.\n";
+ String copyFromLocal = "-copyFromLocal <localsrc> ... <dst>:" +
+ " Identical to the -put command.\n";
- String moveFromLocal = "-moveFromLocal <localsrc> <dst>: Same as -put, except that the source is\n" +
- "\t\tdeleted after it's copied.\n";
+ String moveFromLocal = "-moveFromLocal <localsrc> ... <dst>:" +
+ " Same as -put, except that the source is\n\t\tdeleted after it's copied.\n";
String get = GET_SHORT_USAGE
+ ": Copy files that match the file pattern <src> \n" +
@@ -1618,9 +1637,9 @@
System.err.println(" [-rm <path>]");
System.err.println(" [-rmr <path>]");
System.err.println(" [-expunge]");
- System.err.println(" [-put <localsrc> <dst>]");
- System.err.println(" [-copyFromLocal <localsrc> <dst>]");
- System.err.println(" [-moveFromLocal <localsrc> <dst>]");
+ System.err.println(" [-put <localsrc> ... <dst>]");
+ System.err.println(" [-copyFromLocal <localsrc> ... <dst>]");
+ System.err.println(" [-moveFromLocal <localsrc> ... <dst>]");
System.err.println(" [" + GET_SHORT_USAGE + "]");
System.err.println(" [-getmerge <src> <localdst> [addnl]]");
System.err.println(" [-cat <src>]");
@@ -1661,7 +1680,7 @@
//
if ("-put".equals(cmd) || "-test".equals(cmd) ||
"-copyFromLocal".equals(cmd) || "-moveFromLocal".equals(cmd)) {
- if (argv.length != 3) {
+ if (argv.length < 3) {
printUsage(cmd);
return exitCode;
}
@@ -1701,9 +1720,15 @@
exitCode = 0;
try {
if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
- copyFromLocal(new Path(argv[i++]), argv[i++]);
+ Path[] srcs = new Path[argv.length-2];
+ for (int j=0 ; i < argv.length-1 ;)
+ srcs[j++] = new Path(argv[i++]);
+ copyFromLocal(srcs, argv[i++]);
} else if ("-moveFromLocal".equals(cmd)) {
- moveFromLocal(new Path(argv[i++]), argv[i++]);
+ Path[] srcs = new Path[argv.length-2];
+ for (int j=0 ; i < argv.length-1 ;)
+ srcs[j++] = new Path(argv[i++]);
+ moveFromLocal(srcs, argv[i++]);
} else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
copyToLocal(argv, i);
} else if ("-getmerge".equals(cmd)) {
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java?rev=641681&r1=641680&r2=641681&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestDFSShell.java Wed Mar 26 18:22:59 2008
@@ -194,6 +194,30 @@
try {copy2ndFileThread.join();} catch (InterruptedException e) { }
System.setSecurityManager(sm);
+
+ // copy multiple files to destination directory
+ final Path destmultiple = mkdir(dfs, new Path("/test/putmultiple"));
+ Path[] srcs = new Path[2];
+ srcs[0] = new Path(f1.getPath());
+ srcs[1] = new Path(f2.getPath());
+ dfs.copyFromLocalFile(false, false, srcs, destmultiple);
+ srcs[0] = new Path(destmultiple,"f1");
+ srcs[1] = new Path(destmultiple,"f2");
+ assertTrue(dfs.exists(srcs[0]));
+ assertTrue(dfs.exists(srcs[1]));
+
+ // move multiple files to destination directory
+ final Path destmultiple2 = mkdir(dfs, new Path("/test/movemultiple"));
+ srcs[0] = new Path(f1.getPath());
+ srcs[1] = new Path(f2.getPath());
+ dfs.moveFromLocalFile(srcs, destmultiple2);
+ assertFalse(dfs.exists(srcs[0]));
+ assertFalse(dfs.exists(srcs[1]));
+ srcs[0] = new Path(destmultiple2, "f1");
+ srcs[1] = new Path(destmultiple2, "f2");
+ assertTrue(dfs.exists(srcs[0]));
+ assertTrue(dfs.exists(srcs[1]));
+
f1.delete();
f2.delete();
} finally {