You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2011/05/24 22:33:40 UTC
svn commit: r1127271 - in /hadoop/common/branches/yahoo-merge: ./
src/contrib/ec2/ src/docs/ src/java/ src/java/org/apache/hadoop/fs/
src/java/org/apache/hadoop/fs/shell/ src/test/core/
src/test/core/org/apache/hadoop/cli/
Author: szetszwo
Date: Tue May 24 20:33:39 2011
New Revision: 1127271
URL: http://svn.apache.org/viewvc?rev=1127271&view=rev
Log:
svn merge -c 1100369 from trunk for HADOOP-7251.
Added:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Copy.java
- copied unchanged from r1100369, hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Copy.java
Modified:
hadoop/common/branches/yahoo-merge/ (props changed)
hadoop/common/branches/yahoo-merge/CHANGES.txt (contents, props changed)
hadoop/common/branches/yahoo-merge/src/contrib/ec2/ (props changed)
hadoop/common/branches/yahoo-merge/src/docs/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
hadoop/common/branches/yahoo-merge/src/test/core/ (props changed)
hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml
Propchange: hadoop/common/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:33:39 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100400
+/hadoop/common/trunk:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400
/hadoop/core/branches/branch-0.19/core:713112
Modified: hadoop/common/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/CHANGES.txt?rev=1127271&r1=1127270&r2=1127271&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/common/branches/yahoo-merge/CHANGES.txt Tue May 24 20:33:39 2011
@@ -50,6 +50,9 @@ Trunk (unreleased changes)
HADOOP-7249. Refactor the chmod/chown/chgrp command to conform to new
FsCommand class. (Daryn Sharp via szetszwo)
+ HADOOP-7251. Refactor the getmerge command to conform to new FsCommand
+ class. (Daryn Sharp via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
Propchange: hadoop/common/branches/yahoo-merge/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:33:39 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/CHANGES.txt:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1092832,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100400
+/hadoop/common/trunk/CHANGES.txt:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1092832,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/yahoo-merge/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:33:39 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/contrib/ec2:1043117,1076296,1078148,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100400
+/hadoop/common/trunk/src/contrib/ec2:1043117,1076296,1078148,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400
/hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
/hadoop/core/trunk/src/contrib/ec2:776175-784663
Propchange: hadoop/common/branches/yahoo-merge/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:33:39 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/src/docs:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100400
+/hadoop/common/trunk/src/docs:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange: hadoop/common/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:33:39 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/java:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100400
+/hadoop/common/trunk/src/java:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java?rev=1127271&r1=1127270&r2=1127271&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java Tue May 24 20:33:39 2011
@@ -308,49 +308,6 @@ public class FsShell extends Configured
}
/**
- * Get all the files in the directories that match the source file
- * pattern and merge and sort them to only one file on local fs
- * srcf is kept.
- * @param srcf a file pattern specifying source files
- * @param dstf a destination local file/directory
- * @exception IOException
- * @see org.apache.hadoop.fs.FileSystem.globStatus
- */
- void copyMergeToLocal(String srcf, Path dst) throws IOException {
- copyMergeToLocal(srcf, dst, false);
- }
-
-
- /**
- * Get all the files in the directories that match the source file pattern
- * and merge and sort them to only one file on local fs
- * srcf is kept.
- *
- * Also adds a string between the files (useful for adding \n
- * to a text file)
- * @param srcf a file pattern specifying source files
- * @param dstf a destination local file/directory
- * @param endline if an end of line character is added to a text file
- * @throws IOException on error
- * @see org.apache.hadoop.fs.FileSystem.globStatus
- */
- void copyMergeToLocal(String srcf, Path dst, boolean endline) throws IOException {
- Path srcPath = new Path(srcf);
- FileSystem srcFs = srcPath.getFileSystem(getConf());
- Path [] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath),
- srcPath);
- for(int i=0; i<srcs.length; i++) {
- if (endline) {
- FileUtil.copyMerge(srcFs, srcs[i],
- FileSystem.getLocal(getConf()), dst, false, getConf(), "\n");
- } else {
- FileUtil.copyMerge(srcFs, srcs[i],
- FileSystem.getLocal(getConf()), dst, false, getConf(), null);
- }
- }
- }
-
- /**
* Obtain the indicated file and copy to the local name.
* srcf is removed.
*/
@@ -994,7 +951,7 @@ public class FsShell extends Configured
"[-rmr [-skipTrash] <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" +
"[-moveFromLocal <localsrc> ... <dst>] [" +
GET_SHORT_USAGE + "\n\t" +
- "[-getmerge <src> <localdst> [addnl]] [-cat <src>]\n\t" +
+ "[-cat <src>]\n\t" +
"[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" +
"[-report]\n\t" +
"[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]\n\t" +
@@ -1066,10 +1023,6 @@ public class FsShell extends Configured
"\t\tto the local name. <src> is kept. When copying mutiple, \n" +
"\t\tfiles, the destination must be a directory. \n";
- String getmerge = "-getmerge <src> <localdst>: Get all the files in the directories that \n" +
- "\t\tmatch the source file pattern and merge and sort them to only\n" +
- "\t\tone file on local fs. <src> is kept.\n";
-
String cat = "-cat <src>: \tFetch all files that match the file pattern <src> \n" +
"\t\tand display their content on stdout.\n";
@@ -1132,8 +1085,6 @@ public class FsShell extends Configured
System.out.println(moveFromLocal);
} else if ("get".equals(cmd)) {
System.out.println(get);
- } else if ("getmerge".equals(cmd)) {
- System.out.println(getmerge);
} else if ("copyToLocal".equals(cmd)) {
System.out.println(copyToLocal);
} else if ("moveToLocal".equals(cmd)) {
@@ -1172,7 +1123,6 @@ public class FsShell extends Configured
System.out.println(copyFromLocal);
System.out.println(moveFromLocal);
System.out.println(get);
- System.out.println(getmerge);
System.out.println(cat);
System.out.println(copyToLocal);
System.out.println(moveToLocal);
@@ -1342,7 +1292,6 @@ public class FsShell extends Configured
System.err.println(" [-copyFromLocal <localsrc> ... <dst>]");
System.err.println(" [-moveFromLocal <localsrc> ... <dst>]");
System.err.println(" [" + GET_SHORT_USAGE + "]");
- System.err.println(" [-getmerge <src> <localdst> [addnl]]");
System.err.println(" [-cat <src>]");
System.err.println(" [-text <src>]");
System.err.println(" [" + COPYTOLOCAL_SHORT_USAGE + "]");
@@ -1448,11 +1397,6 @@ public class FsShell extends Configured
moveFromLocal(srcs, argv[i++]);
} else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
copyToLocal(argv, i);
- } else if ("-getmerge".equals(cmd)) {
- if (argv.length>i+2)
- copyMergeToLocal(argv[i++], new Path(argv[i++]), Boolean.parseBoolean(argv[i++]));
- else
- copyMergeToLocal(argv[i++], new Path(argv[i++]));
} else if ("-cat".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-text".equals(cmd)) {
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java?rev=1127271&r1=1127270&r2=1127271&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java Tue May 24 20:33:39 2011
@@ -43,6 +43,7 @@ abstract public class FsCommand extends
* @param factory where to register the class
*/
public static void registerCommands(CommandFactory factory) {
+ factory.registerCommands(Copy.class);
factory.registerCommands(Count.class);
factory.registerCommands(FsShellPermissions.class);
factory.registerCommands(Ls.class);
Propchange: hadoop/common/branches/yahoo-merge/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:33:39 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/test/core:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100400
+/hadoop/common/trunk/src/test/core:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml?rev=1127271&r1=1127270&r2=1127271&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml (original)
+++ hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml Tue May 24 20:33:39 2011
@@ -430,7 +430,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-getmerge <src> <localdst>: Get all the files in the directories that( )*</expected-output>
+ <expected-output>^-getmerge <src> <localdst> \[addnl\]:( |\t)*Get all the files in the directories that( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>