You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2018/10/11 19:11:01 UTC
[1/2] hadoop git commit: MAPREDUCE-7149. Javadocs for FileInputFormat
and OutputFormat to mention DT collection. Contributed by Steve Loughran.
Repository: hadoop
Updated Branches:
refs/heads/trunk 96d28b475 -> 2addebb94
MAPREDUCE-7149. Javadocs for FileInputFormat and OutputFormat to mention DT collection.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/735a36af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/735a36af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/735a36af
Branch: refs/heads/trunk
Commit: 735a36afd528862d7690bee2d71b2f60dd282d21
Parents: 96d28b4
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 20:01:42 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 20:01:42 2018 +0100
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapred/FileInputFormat.java | 8 ++++++--
.../main/java/org/apache/hadoop/mapred/OutputFormat.java | 3 +++
.../java/org/apache/hadoop/mapreduce/OutputFormat.java | 3 +++
.../hadoop/mapreduce/lib/input/FileInputFormat.java | 10 +++++++---
4 files changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index fe43991..afdc0ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -198,11 +198,15 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
}
}
- /** List input directories.
+ /**
+ * List input directories.
* Subclasses may override to, e.g., select only files matching a regular
* expression.
*
- * @param job the job to list input paths for
+ * If security is enabled, this method collects
+ * delegation tokens from the input paths and adds them to the job's
+ * credentials.
+ * @param job the job to list input paths for and attach tokens to.
* @return array of FileStatus objects
* @throws IOException if zero items.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
index 6ea6a71..adfda0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/OutputFormat.java
@@ -71,6 +71,9 @@ public interface OutputFormat<K, V> {
* throwing an exception when it already exists, so that output is not
* overwritten.</p>
*
+ * Implementations which write to filesystems which support delegation
+ * tokens usually collect the tokens for the destination path(s)
+ * and attach them to the job configuration.
* @param ignored
* @param job job configuration.
* @throws IOException when output should not be attempted
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
index 6f9e525..901d5ee 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/OutputFormat.java
@@ -66,6 +66,9 @@ public abstract class OutputFormat<K, V> {
* throwing an exception when it already exists, so that output is not
* overwritten.</p>
*
+ * Implementations which write to filesystems which support delegation
+ * tokens usually collect the tokens for the destination path(s)
+ * and attach them to the job context's JobConf.
* @param context information about the job
* @throws IOException when output should not be attempted
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/735a36af/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index e2d8e6f..e2658ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -233,11 +233,15 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
(PathFilter) ReflectionUtils.newInstance(filterClass, conf) : null;
}
- /** List input directories.
+ /**
+ * List input directories.
* Subclasses may override to, e.g., select only files matching a regular
* expression.
- *
- * @param job the job to list input paths for
+ *
+ * If security is enabled, this method collects
+ * delegation tokens from the input paths and adds them to the job's
+ * credentials.
+ * @param job the job to list input paths for and attach tokens to.
* @return array of FileStatus objects
* @throws IOException if zero items.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: HDFS-13951. HDFS DelegationTokenFetcher
can't print non-HDFS tokens in a tokenfile. Contributed by Steve Loughran.
Posted by st...@apache.org.
HDFS-13951. HDFS DelegationTokenFetcher can't print non-HDFS tokens in a tokenfile.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2addebb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2addebb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2addebb9
Branch: refs/heads/trunk
Commit: 2addebb94f592e46b261e2edd9e95a82e83bd761
Parents: 735a36a
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 20:03:56 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 20:03:56 2018 +0100
----------------------------------------------------------------------
.../hdfs/tools/DelegationTokenFetcher.java | 58 +++++++++++++++-----
1 file changed, 43 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2addebb9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index 7d1e88f..0dc4bcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.tools;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
@@ -36,12 +34,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.GenericOptionsParser;
@@ -64,10 +62,29 @@ public class DelegationTokenFetcher {
private static final String RENEWER = "renewer";
private static final String VERBOSE = "verbose";
/**
- * Command-line interface
+ * Command-line interface.
+ * @param args argument list.
+ * @throws Exception on a failure.
+ * @throws org.apache.hadoop.util.ExitUtil.ExitException if the command
+ * failed and exiting was disabled.
*/
public static void main(final String[] args) throws Exception {
- final Configuration conf = new HdfsConfiguration();
+ main(new HdfsConfiguration(), args);
+ }
+
+ /**
+ * Command line interface with a specific configuration.
+ * Errors in this operation will call {@link ExitUtil#terminate(int)} to
+ * exit the process.
+ * @param conf configuration to create filesystems with.
+ * @param args argument list.
+ * @throws Exception on a failure.
+ * @throws org.apache.hadoop.util.ExitUtil.ExitException if the command
+ * failed and exiting was disabled.
+ */
+ @VisibleForTesting
+ public static void main(Configuration conf, final String[] args)
+ throws Exception {
Options fetcherOptions = new Options();
fetcherOptions
.addOption(WEBSERVICE, true, "HTTP url to reach the NameNode at")
@@ -96,17 +113,19 @@ public class DelegationTokenFetcher {
// check option validity
if (help) {
printUsage(System.out);
- System.exit(0);
+ return;
}
int commandCount = (cancel ? 1 : 0) + (renew ? 1 : 0) + (print ? 1 : 0);
if (commandCount > 1) {
System.err.println("ERROR: Only specify cancel, renew or print.");
printUsage(System.err);
+ return;
}
if (remaining.length != 1 || remaining[0].charAt(0) == '-') {
System.err.println("ERROR: Must specify exactly one token file");
printUsage(System.err);
+ return;
}
// default to using the local file system
FileSystem local = FileSystem.getLocal(conf);
@@ -195,19 +214,22 @@ public class DelegationTokenFetcher {
}
@VisibleForTesting
- static String printTokensToString(
+ public static String printTokensToString(
final Configuration conf,
final Path tokenFile,
final boolean verbose) throws IOException {
StringBuilder sbld = new StringBuilder();
final String nl = System.getProperty("line.separator");
- DelegationTokenIdentifier id = new DelegationTokenSecretManager(0, 0, 0,
- 0, null).createIdentifier();
for (Token<?> token : readTokens(tokenFile, conf)) {
- DataInputStream in = new DataInputStream(new ByteArrayInputStream(token
- .getIdentifier()));
- id.readFields(in);
- String idStr = (verbose? id.toString() : id.toStringStable());
+ TokenIdentifier tokenId = token.decodeIdentifier();
+
+ String idStr;
+ if (tokenId instanceof DelegationTokenIdentifier) {
+ DelegationTokenIdentifier id = (DelegationTokenIdentifier) tokenId;
+ idStr = (verbose? id.toString() : id.toStringStable());
+ } else {
+ idStr = tokenId.toString();
+ }
sbld
.append("Token (").append(idStr)
.append(") for ").append(token.getService()).append(nl);
@@ -222,13 +244,19 @@ public class DelegationTokenFetcher {
System.out.print(printTokensToString(conf, tokenFile, verbose));
}
+ /**
+ * Print usage to the error stream, then
+ * call {@link ExitUtil#terminate(int)} with status code 1.
+ * This will exit or raise an exception if that's been disabled.
+ * @param err stream for the messages.
+ */
private static void printUsage(PrintStream err) {
err.println("fetchdt retrieves delegation tokens from the NameNode");
err.println();
err.println("fetchdt <opts> <token file>");
err.println("Options:");
- err.println(" --webservice <url> Url to contact NN on (starts with " +
- "http:// or https://)");
+ err.println(" --webservice <url> URL to contact NN on (starts with " +
+ "http:// or https://), or other filesystem URL");
err.println(" --renewer <name> Name of the delegation token renewer");
err.println(" --cancel Cancel the delegation token");
err.println(" --renew Renew the delegation token. " +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org