You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/11/04 15:23:00 UTC
[14/51] [abbrv] hadoop git commit: HDFS-206. Support for head in
FSShell. Contributed by Gabor Bota.
HDFS-206. Support for head in FSShell. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81225430
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81225430
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81225430
Branch: refs/heads/YARN-6592
Commit: 81225430d0153e8515f2ecb56ece7fcfe289d5ee
Parents: 2fae63a
Author: Sean Mackrory <ma...@apache.org>
Authored: Mon Oct 30 16:35:42 2017 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Tue Oct 31 08:34:01 2017 -0600
----------------------------------------------------------------------
.../org/apache/hadoop/fs/shell/FsCommand.java | 1 +
.../java/org/apache/hadoop/fs/shell/Head.java | 78 ++++++++++++++++++++
.../src/site/markdown/FileSystemShell.md | 13 ++++
.../org/apache/hadoop/hdfs/TestDFSShell.java | 27 +++++++
4 files changed, 119 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81225430/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
index f274f67..4a13414 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
@@ -64,6 +64,7 @@ abstract public class FsCommand extends Command {
factory.registerCommands(SetReplication.class);
factory.registerCommands(Stat.class);
factory.registerCommands(Tail.class);
+ factory.registerCommands(Head.class);
factory.registerCommands(Test.class);
factory.registerCommands(Touch.class);
factory.registerCommands(Truncate.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81225430/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
new file mode 100644
index 0000000..2280225
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Head.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.shell;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.PathIsDirectoryException;
+import org.apache.hadoop.io.IOUtils;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * Show the first 1KB of the file.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+
+class Head extends FsCommand {
+ public static void registerCommands(CommandFactory factory) {
+ factory.addClass(Head.class, "-head");
+ }
+ public static final String NAME = "head";
+ public static final String USAGE = "<file>";
+ public static final String DESCRIPTION =
+ "Show the first 1KB of the file.\n";
+
+ private long endingOffset = 1024;
+
+ @Override
+ protected void processOptions(LinkedList<String> args) throws IOException {
+ CommandFormat cf = new CommandFormat(1, 1);
+ cf.parse(args);
+ }
+
+ @Override
+ protected List<PathData> expandArgument(String arg) throws IOException {
+ List<PathData> items = new LinkedList<PathData>();
+ items.add(new PathData(arg, getConf()));
+ return items;
+ }
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ if (item.stat.isDirectory()) {
+ throw new PathIsDirectoryException(item.toString());
+ }
+
+ dumpToOffset(item);
+ }
+
+ private void dumpToOffset(PathData item) throws IOException {
+ FSDataInputStream in = item.fs.open(item.path);
+ try {
+ IOUtils.copyBytes(in, System.out, endingOffset, false);
+ } finally {
+ in.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81225430/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 71eec75..ec9d3c3 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -400,6 +400,19 @@ Exit Code:
Returns 0 on success and non-zero on error.
+head
+----
+
+Usage: `hadoop fs -head URI`
+
+Displays first kilobyte of the file to stdout.
+
+Example:
+
+* `hadoop fs -head pathname`
+
+Exit Code: Returns 0 on success and -1 on error.
+
help
----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/81225430/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 9ae49aa..e82863a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -893,6 +893,33 @@ public class TestDFSShell {
}
/**
+ * Test that -head displays first kilobyte of the file to stdout.
+ */
+ @Test (timeout = 30000)
+ public void testHead() throws Exception {
+ final int fileLen = 5 * BLOCK_SIZE;
+
+ // create a text file with multiple KB bytes (and multiple blocks)
+ final Path testFile = new Path("testHead", "file1");
+ final String text = RandomStringUtils.randomAscii(fileLen);
+ try (OutputStream pout = dfs.create(testFile)) {
+ pout.write(text.getBytes());
+ }
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ System.setOut(new PrintStream(out));
+ final String[] argv = new String[]{"-head", testFile.toString()};
+ final int ret = ToolRunner.run(new FsShell(dfs.getConf()), argv);
+
+ assertEquals(Arrays.toString(argv) + " returned " + ret, 0, ret);
+ assertEquals("-head returned " + out.size() + " bytes data, expected 1KB",
+ 1024, out.size());
+ // tailed out last 1KB of the file content
+ assertArrayEquals("Head output doesn't match input",
+ text.substring(0, 1024).getBytes(), out.toByteArray());
+ out.reset();
+ }
+
+ /**
* Test that -tail displays last kilobyte of the file to stdout.
*/
@Test (timeout = 30000)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org