You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2011/04/27 06:13:23 UTC
svn commit: r1096988 - in /hadoop/common/trunk: ./
src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/fs/shell/
src/test/core/org/apache/hadoop/cli/
Author: szetszwo
Date: Wed Apr 27 04:13:23 2011
New Revision: 1096988
URL: http://svn.apache.org/viewvc?rev=1096988&view=rev
Log:
HADOOP-7235. Refactor the tail command to conform to new FsCommand class. Contributed by Daryn Sharp
Added:
hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Tail.java
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java
hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/FsCommand.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/testConf.xml
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1096988&r1=1096987&r2=1096988&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Wed Apr 27 04:13:23 2011
@@ -99,6 +99,9 @@ Trunk (unreleased changes)
HADOOP-7233. Refactor ls to conform to new FsCommand class. (Daryn Sharp
via szetszwo)
+ HADOOP-7235. Refactor the tail command to conform to new FsCommand class.
+ (Daryn Sharp via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java?rev=1096988&r1=1096987&r2=1096988&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/FsShell.java Wed Apr 27 04:13:23 2011
@@ -76,7 +76,6 @@ public class FsShell extends Configured
static final String GET_SHORT_USAGE = "-get [-ignoreCrc] [-crc] <src> <localdst>";
static final String COPYTOLOCAL_SHORT_USAGE = GET_SHORT_USAGE.replace(
"-get", "-copyToLocal");
- static final String TAIL_USAGE="-tail [-f] <file>";
static final String DU_USAGE="-du [-s] [-h] <paths...>";
/**
@@ -1140,57 +1139,6 @@ public class FsShell extends Configured
}
/**
- * Parse the incoming command string
- * @param cmd
- * @param pos ignore anything before this pos in cmd
- * @throws IOException
- */
- private void tail(String[] cmd, int pos) throws IOException {
- CommandFormat c = new CommandFormat("tail", 1, 1, "f");
- String src = null;
- Path path = null;
-
- try {
- List<String> parameters = c.parse(cmd, pos);
- src = parameters.get(0);
- } catch(IllegalArgumentException iae) {
- System.err.println("Usage: java FsShell " + TAIL_USAGE);
- throw iae;
- }
- boolean foption = c.getOpt("f") ? true: false;
- path = new Path(src);
- FileSystem srcFs = path.getFileSystem(getConf());
- FileStatus fileStatus = srcFs.getFileStatus(path);
- if (fileStatus.isDirectory()) {
- throw new IOException("Source must be a file.");
- }
-
- long fileSize = fileStatus.getLen();
- long offset = (fileSize > 1024) ? fileSize - 1024: 0;
-
- while (true) {
- FSDataInputStream in = srcFs.open(path);
- try {
- in.seek(offset);
- IOUtils.copyBytes(in, System.out, 1024);
- offset = in.getPos();
- } finally {
- in.close();
- }
- if (!foption) {
- break;
- }
- fileSize = srcFs.getFileStatus(path).getLen();
- offset = (fileSize > offset) ? offset: fileSize;
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- break;
- }
- }
- }
-
- /**
* This class runs a command on a given FileStatus. This can be used for
* running various commands like chmod, chown etc.
*/
@@ -1328,7 +1276,7 @@ public class FsShell extends Configured
"[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" +
"[-mkdir <path>] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" +
"[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]\n\t" +
- "[-tail [-f] <path>] [-text <path>]\n\t" +
+ "[-text <path>]\n\t" +
"[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" +
"[" + FsShellPermissions.CHOWN_USAGE + "]\n\t" +
"[" + FsShellPermissions.CHGRP_USAGE + "]";
@@ -1434,10 +1382,6 @@ public class FsShell extends Configured
"\t\tin the specified format. Format accepts filesize in blocks (%b), filename (%n),\n" +
"\t\tblock size (%o), replication (%r), modification date (%y, %Y)\n";
- String tail = TAIL_USAGE
- + ": Show the last 1KB of the file. \n"
- + "\t\tThe -f option shows appended data as the file grows. \n";
-
String chmod = FsShellPermissions.CHMOD_USAGE + "\n" +
"\t\tChanges permissions of a file.\n" +
"\t\tThis works similar to shell's chmod with a few exceptions.\n\n" +
@@ -1530,8 +1474,6 @@ public class FsShell extends Configured
System.out.println(text);
} else if ("stat".equals(cmd)) {
System.out.println(stat);
- } else if ("tail".equals(cmd)) {
- System.out.println(tail);
} else if ("chmod".equals(cmd)) {
System.out.println(chmod);
} else if ("chown".equals(cmd)) {
@@ -1566,7 +1508,6 @@ public class FsShell extends Configured
System.out.println(moveToLocal);
System.out.println(mkdir);
System.out.println(setrep);
- System.out.println(tail);
System.out.println(touchz);
System.out.println(test);
System.out.println(text);
@@ -1726,8 +1667,6 @@ public class FsShell extends Configured
} else if ("-stat".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-stat [format] <path>]");
- } else if ("-tail".equals(cmd)) {
- System.err.println("Usage: java FsShell [" + TAIL_USAGE + "]");
} else {
System.err.println("Usage: java FsShell");
System.err.println(" [-df [<path>]]");
@@ -1752,7 +1691,6 @@ public class FsShell extends Configured
System.err.println(" [-touchz <path>]");
System.err.println(" [-test -[ezd] <path>]");
System.err.println(" [-stat [format] <path>]");
- System.err.println(" [" + TAIL_USAGE + "]");
System.err.println(" [" + FsShellPermissions.CHMOD_USAGE + "]");
System.err.println(" [" + FsShellPermissions.CHOWN_USAGE + "]");
System.err.println(" [" + FsShellPermissions.CHGRP_USAGE + "]");
@@ -1909,8 +1847,6 @@ public class FsShell extends Configured
} else {
printHelp("");
}
- } else if ("-tail".equals(cmd)) {
- tail(argv, i);
} else {
exitCode = -1;
System.err.println(cmd.substring(1) + ": Unknown command");
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java?rev=1096988&r1=1096987&r2=1096988&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Command.java Wed Apr 27 04:13:23 2011
@@ -259,12 +259,13 @@ abstract public class Command extends Co
/**
* TODO: A crutch until the text is standardized across commands...
* Eventually an exception that takes the path as an argument will
- * replace custom text
+ * replace custom text, until then, commands can supply custom text
+ * for backwards compatibility
* @param path the thing that doesn't exist
* @returns String in printf format
*/
protected String getFnfText(Path path) {
- throw new RuntimeException(path + ": No such file or directory");
+ return path + ": No such file or directory";
}
/**
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/FsCommand.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/FsCommand.java?rev=1096988&r1=1096987&r2=1096988&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/FsCommand.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/FsCommand.java Wed Apr 27 04:13:23 2011
@@ -44,6 +44,7 @@ abstract public class FsCommand extends
public static void registerCommands(CommandFactory factory) {
factory.registerCommands(Count.class);
factory.registerCommands(Ls.class);
+ factory.registerCommands(Tail.class);
}
protected FsCommand() {}
Added: hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Tail.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Tail.java?rev=1096988&view=auto
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Tail.java (added)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/shell/Tail.java Wed Apr 27 04:13:23 2011
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * Get a listing of all files in that match the file patterns.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+
+class Tail extends FsCommand {
+ public static void registerCommands(CommandFactory factory) {
+ factory.addClass(Tail.class, "-tail");
+ }
+
+ public static final String NAME = "tail";
+ public static final String USAGE = "[-f] <file>";
+ public static final String DESCRIPTION =
+ "Show the last 1KB of the file.\n" +
+ "\t\tThe -f option shows appended data as the file grows.\n";
+
+ private long startingOffset = -1024;
+ private boolean follow = false;
+ private long followDelay = 5000; // milliseconds
+
+ @Override
+ protected void processOptions(LinkedList<String> args) throws IOException {
+ CommandFormat cf = new CommandFormat(null, 1, 1, "f");
+ cf.parse(args);
+ follow = cf.getOpt("f");
+ }
+
+ // TODO: HADOOP-7234 will add glob support; for now, be backwards compat
+ @Override
+ protected List<PathData> expandArgument(String arg) throws IOException {
+ Path path = new Path(arg);
+ FileSystem fs = path.getFileSystem(getConf());
+
+ List<PathData> items = new LinkedList<PathData>();
+ items.add(new PathData(fs, path));
+ return items;
+ }
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ if (item.stat.isDirectory()) {
+ throw new IOException("Source must be a file.");
+ }
+
+ long offset = dumpFromOffset(item, startingOffset);
+ while (follow) {
+ try {
+ Thread.sleep(followDelay);
+ } catch (InterruptedException e) {
+ break;
+ }
+ offset = dumpFromOffset(item, offset);
+ }
+ }
+
+ private long dumpFromOffset(PathData item, long offset) throws IOException {
+ long fileSize = item.refreshStatus().getLen();
+ if (offset > fileSize) return fileSize;
+ // treat a negative offset as relative to end of the file, floor of 0
+ if (offset < 0) {
+ offset = Math.max(fileSize + offset, 0);
+ }
+
+ FSDataInputStream in = item.fs.open(item.path);
+ try {
+ in.seek(offset);
+ // use conf so the system configured io block size is used
+ IOUtils.copyBytes(in, System.out, getConf(), false);
+ offset = in.getPos();
+ } finally {
+ in.close();
+ }
+ return offset;
+ }
+
+ @Override
+ protected String getFnfText(Path path) {
+ return "File does not exist: " + path;
+ }
+}
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/testConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/testConf.xml?rev=1096988&r1=1096987&r2=1096988&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/testConf.xml (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/cli/testConf.xml Wed Apr 27 04:13:23 2011
@@ -606,7 +606,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-tail \[-f\] <file>: Show the last 1KB of the file.( )*</expected-output>
+ <expected-output>^-tail \[-f\] <file>:( |\t)+Show the last 1KB of the file.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>