You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2011/05/24 22:56:52 UTC
svn commit: r1127280 - in /hadoop/common/branches/yahoo-merge: ./
src/contrib/ec2/ src/docs/ src/java/ src/java/org/apache/hadoop/fs/
src/java/org/apache/hadoop/fs/shell/ src/test/core/
src/test/core/org/apache/hadoop/cli/
Author: szetszwo
Date: Tue May 24 20:56:51 2011
New Revision: 1127280
URL: http://svn.apache.org/viewvc?rev=1127280&view=rev
Log:
svn merge -c 1101199 from trunk for HADOOP-7238.
Added:
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/Display.java
- copied unchanged from r1101199, hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Display.java
Modified:
hadoop/common/branches/yahoo-merge/ (props changed)
hadoop/common/branches/yahoo-merge/CHANGES.txt (contents, props changed)
hadoop/common/branches/yahoo-merge/src/contrib/ec2/ (props changed)
hadoop/common/branches/yahoo-merge/src/docs/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
hadoop/common/branches/yahoo-merge/src/test/core/ (props changed)
hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml
Propchange: hadoop/common/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:56:51 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132
+/hadoop/common/trunk:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199
/hadoop/core/branches/branch-0.19/core:713112
Modified: hadoop/common/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/CHANGES.txt?rev=1127280&r1=1127279&r2=1127280&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/common/branches/yahoo-merge/CHANGES.txt Tue May 24 20:56:51 2011
@@ -56,6 +56,9 @@ Trunk (unreleased changes)
HADOOP-7265. Keep track of relative paths in PathData. (Daryn Sharp
via szetszwo)
+ HADOOP-7238. Refactor the cat and text commands to conform to new FsCommand
+ class. (Daryn Sharp via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
Propchange: hadoop/common/branches/yahoo-merge/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:56:51 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/CHANGES.txt:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1092832,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132
+/hadoop/common/trunk/CHANGES.txt:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1092832,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/yahoo-merge/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:56:51 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/contrib/ec2:1043117,1076296,1078148,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132
+/hadoop/common/trunk/src/contrib/ec2:1043117,1076296,1078148,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199
/hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
/hadoop/core/trunk/src/contrib/ec2:776175-784663
Propchange: hadoop/common/branches/yahoo-merge/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:56:51 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/src/docs:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132
+/hadoop/common/trunk/src/docs:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange: hadoop/common/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:56:51 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/java:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132
+/hadoop/common/trunk/src/java:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java?rev=1127280&r1=1127279&r2=1127280&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FsShell.java Tue May 24 20:56:51 2011
@@ -28,7 +28,6 @@ import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.TimeZone;
-import java.util.zip.GZIPInputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -39,17 +38,9 @@ import org.apache.hadoop.fs.shell.Comman
import org.apache.hadoop.fs.shell.CommandFactory;
import org.apache.hadoop.fs.shell.CommandFormat;
import org.apache.hadoop.fs.shell.FsCommand;
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -173,12 +164,12 @@ public class FsShell extends Configured
* and copy them to the local name. srcf is kept.
* When copying multiple files, the destination must be a directory.
* Otherwise, IOException is thrown.
- * @param argv arguments
- * @param pos Ignore everything before argv[pos]
- * @throws IOException on error
+ * @param argv : arguments
+ * @param pos : Ignore everything before argv[pos]
+ * @throws Exception
* @see org.apache.hadoop.fs.FileSystem.globStatus
*/
- void copyToLocal(String[]argv, int pos) throws IOException {
+ void copyToLocal(String[]argv, int pos) throws Exception {
CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
String srcstr = null;
@@ -199,7 +190,12 @@ public class FsShell extends Configured
if (copyCrc) {
System.err.println("-crc option is not valid when destination is stdout.");
}
- cat(srcstr, verifyChecksum);
+
+ List<String> catArgv = new ArrayList<String>();
+ catArgv.add("-cat");
+ if (cf.getOpt("ignoreCrc")) catArgv.add("-ignoreCrc");
+ catArgv.add(srcstr);
+ run(catArgv.toArray(new String[0]));
} else {
File dst = new File(dststr);
Path srcpath = new Path(srcstr);
@@ -314,115 +310,6 @@ public class FsShell extends Configured
void moveToLocal(String srcf, Path dst) throws IOException {
System.err.println("Option '-moveToLocal' is not implemented yet.");
}
-
- /**
- * Fetch all files that match the file pattern <i>srcf</i> and display
- * their content on stdout.
- * @param srcf a file pattern specifying source files
- * @throws IOException on error
- * @see org.apache.hadoop.fs.FileSystem.globStatus
- */
- void cat(String src, boolean verifyChecksum) throws IOException {
- //cat behavior in Linux
- // [~/1207]$ ls ?.txt
- // x.txt z.txt
- // [~/1207]$ cat x.txt y.txt z.txt
- // xxx
- // cat: y.txt: No such file or directory
- // zzz
-
- Path srcPattern = new Path(src);
- new DelayedExceptionThrowing() {
- @Override
- void process(Path p, FileSystem srcFs) throws IOException {
- printToStdout(srcFs.open(p));
- }
- }.globAndProcess(srcPattern, getSrcFileSystem(srcPattern, verifyChecksum));
- }
-
- private class TextRecordInputStream extends InputStream {
- SequenceFile.Reader r;
- WritableComparable<?> key;
- Writable val;
-
- DataInputBuffer inbuf;
- DataOutputBuffer outbuf;
-
- public TextRecordInputStream(FileStatus f) throws IOException {
- final Path fpath = f.getPath();
- final Configuration lconf = getConf();
- r = new SequenceFile.Reader(lconf,
- SequenceFile.Reader.file(fpath));
- key = ReflectionUtils.newInstance(
- r.getKeyClass().asSubclass(WritableComparable.class), lconf);
- val = ReflectionUtils.newInstance(
- r.getValueClass().asSubclass(Writable.class), lconf);
- inbuf = new DataInputBuffer();
- outbuf = new DataOutputBuffer();
- }
-
- public int read() throws IOException {
- int ret;
- if (null == inbuf || -1 == (ret = inbuf.read())) {
- if (!r.next(key, val)) {
- return -1;
- }
- byte[] tmp = key.toString().getBytes();
- outbuf.write(tmp, 0, tmp.length);
- outbuf.write('\t');
- tmp = val.toString().getBytes();
- outbuf.write(tmp, 0, tmp.length);
- outbuf.write('\n');
- inbuf.reset(outbuf.getData(), outbuf.getLength());
- outbuf.reset();
- ret = inbuf.read();
- }
- return ret;
- }
-
- public void close() throws IOException {
- r.close();
- super.close();
- }
- }
-
- private InputStream forMagic(Path p, FileSystem srcFs) throws IOException {
- FSDataInputStream i = srcFs.open(p);
-
- // check codecs
- CompressionCodecFactory cf = new CompressionCodecFactory(getConf());
- CompressionCodec codec = cf.getCodec(p);
- if (codec != null) {
- return codec.createInputStream(i);
- }
-
- switch(i.readShort()) {
- case 0x1f8b: // RFC 1952
- i.seek(0);
- return new GZIPInputStream(i);
- case 0x5345: // 'S' 'E'
- if (i.readByte() == 'Q') {
- i.close();
- return new TextRecordInputStream(srcFs.getFileStatus(p));
- }
- break;
- }
- i.seek(0);
- return i;
- }
-
- void text(String srcf) throws IOException {
- Path srcPattern = new Path(srcf);
- new DelayedExceptionThrowing() {
- @Override
- void process(Path p, FileSystem srcFs) throws IOException {
- if (srcFs.isDirectory(p)) {
- throw new IOException("Source must be a file.");
- }
- printToStdout(forMagic(p, srcFs));
- }
- }.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf()));
- }
/**
* Show the size of a partition in the filesystem that contains
@@ -951,11 +838,9 @@ public class FsShell extends Configured
"[-rmr [-skipTrash] <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" +
"[-moveFromLocal <localsrc> ... <dst>] [" +
GET_SHORT_USAGE + "\n\t" +
- "[-cat <src>]\n\t" +
"[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" +
"[-report]\n\t" +
- "[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]\n\t" +
- "[-text <path>]";
+ "[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]";
String conf ="-conf <configuration file>: Specify an application configuration file.";
@@ -1023,14 +908,6 @@ public class FsShell extends Configured
"\t\tto the local name. <src> is kept. When copying mutiple, \n" +
"\t\tfiles, the destination must be a directory. \n";
- String cat = "-cat <src>: \tFetch all files that match the file pattern <src> \n" +
- "\t\tand display their content on stdout.\n";
-
-
- String text = "-text <src>: \tTakes a source file and outputs the file in text format.\n" +
- "\t\tThe allowed formats are zip and TextRecordInputStream.\n";
-
-
String copyToLocal = COPYTOLOCAL_SHORT_USAGE
+ ": Identical to the -get command.\n";
@@ -1089,16 +966,12 @@ public class FsShell extends Configured
System.out.println(copyToLocal);
} else if ("moveToLocal".equals(cmd)) {
System.out.println(moveToLocal);
- } else if ("cat".equals(cmd)) {
- System.out.println(cat);
} else if ("get".equals(cmd)) {
System.out.println(get);
} else if ("touchz".equals(cmd)) {
System.out.println(touchz);
} else if ("test".equals(cmd)) {
System.out.println(test);
- } else if ("text".equals(cmd)) {
- System.out.println(text);
} else if ("stat".equals(cmd)) {
System.out.println(stat);
} else if ("help".equals(cmd)) {
@@ -1123,12 +996,10 @@ public class FsShell extends Configured
System.out.println(copyFromLocal);
System.out.println(moveFromLocal);
System.out.println(get);
- System.out.println(cat);
System.out.println(copyToLocal);
System.out.println(moveToLocal);
System.out.println(touchz);
System.out.println(test);
- System.out.println(text);
System.out.println(stat);
for (String thisCmdName : commandFactory.getNames()) {
@@ -1179,9 +1050,7 @@ public class FsShell extends Configured
//
// issue the command to the fs
//
- if ("-cat".equals(cmd)) {
- cat(argv[i], true);
- } else if ("-rm".equals(cmd)) {
+ if ("-rm".equals(cmd)) {
delete(argv[i], false, rmSkipTrash);
} else if ("-rmr".equals(cmd)) {
delete(argv[i], true, rmSkipTrash);
@@ -1189,8 +1058,6 @@ public class FsShell extends Configured
df(argv[i]);
} else if ("-touchz".equals(cmd)) {
touchz(argv[i]);
- } else if ("-text".equals(cmd)) {
- text(argv[i]);
}
} catch (RemoteException e) {
LOG.debug("Error", e);
@@ -1245,8 +1112,7 @@ public class FsShell extends Configured
System.err.println("Usage: java FsShell" +
" [-D <[property=value>]");
} else if ("-du".equals(cmd) || "-dus".equals(cmd) ||
- "-touchz".equals(cmd) ||
- "-text".equals(cmd)) {
+ "-touchz".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " <path>]");
} else if ("-df".equals(cmd) ) {
@@ -1269,9 +1135,6 @@ public class FsShell extends Configured
} else if ("-moveToLocal".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " [-crc] <src> <localdst>]");
- } else if ("-cat".equals(cmd)) {
- System.err.println("Usage: java FsShell" +
- " [" + cmd + " <src>]");
} else if ("-test".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-test -[ezd] <path>]");
@@ -1292,8 +1155,6 @@ public class FsShell extends Configured
System.err.println(" [-copyFromLocal <localsrc> ... <dst>]");
System.err.println(" [-moveFromLocal <localsrc> ... <dst>]");
System.err.println(" [" + GET_SHORT_USAGE + "]");
- System.err.println(" [-cat <src>]");
- System.err.println(" [-text <src>]");
System.err.println(" [" + COPYTOLOCAL_SHORT_USAGE + "]");
System.err.println(" [-moveToLocal [-crc] <src> <localdst>]");
System.err.println(" [-touchz <path>]");
@@ -1348,9 +1209,7 @@ public class FsShell extends Configured
return exitCode;
}
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd) ||
- "-cat".equals(cmd) ||
- "-touchz".equals(cmd) || "-stat".equals(cmd) ||
- "-text".equals(cmd)) {
+ "-touchz".equals(cmd) || "-stat".equals(cmd)) {
if (argv.length < 2) {
printUsage(cmd);
return exitCode;
@@ -1397,10 +1256,6 @@ public class FsShell extends Configured
moveFromLocal(srcs, argv[i++]);
} else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
copyToLocal(argv, i);
- } else if ("-cat".equals(cmd)) {
- exitCode = doall(cmd, argv, i);
- } else if ("-text".equals(cmd)) {
- exitCode = doall(cmd, argv, i);
} else if ("-moveToLocal".equals(cmd)) {
moveToLocal(argv[i++], new Path(argv[i++]));
} else if ("-mv".equals(cmd)) {
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java?rev=1127280&r1=1127279&r2=1127280&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/shell/FsCommand.java Tue May 24 20:56:51 2011
@@ -45,6 +45,7 @@ abstract public class FsCommand extends
public static void registerCommands(CommandFactory factory) {
factory.registerCommands(Copy.class);
factory.registerCommands(Count.class);
+ factory.registerCommands(Display.class);
factory.registerCommands(FsShellPermissions.class);
factory.registerCommands(Ls.class);
factory.registerCommands(Mkdir.class);
Propchange: hadoop/common/branches/yahoo-merge/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 20:56:51 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/test/core:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132
+/hadoop/common/trunk/src/test/core:1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml?rev=1127280&r1=1127279&r2=1127280&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml (original)
+++ hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/cli/testConf.xml Tue May 24 20:56:51 2011
@@ -454,7 +454,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-cat <src>:( |\t)*Fetch all files that match the file pattern <src>( )*</expected-output>
+ <expected-output>^-cat \[-ignoreCrc\] <src> \.\.\.:( |\t)*Fetch all files that match the file pattern <src>( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>