You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/11/05 20:43:30 UTC
[34/50] [abbrv] hadoop git commit: HADOOP-15885. Add base64
(urlString) support to DTUtil. Contributed by Inigo Goiri.
HADOOP-15885. Add base64 (urlString) support to DTUtil. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/44e37b4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/44e37b4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/44e37b4f
Branch: refs/heads/HDDS-4
Commit: 44e37b4fd9f441becf536368a89436afcd6dede8
Parents: 6d9c18c
Author: Giovanni Matteo Fumarola <gi...@apache.org>
Authored: Fri Nov 2 10:54:12 2018 -0700
Committer: Giovanni Matteo Fumarola <gi...@apache.org>
Committed: Fri Nov 2 10:54:12 2018 -0700
----------------------------------------------------------------------
.../hadoop/security/token/DtFileOperations.java | 28 ++++++++++++-
.../hadoop/security/token/DtUtilShell.java | 37 +++++++++++++++-
.../src/site/markdown/CommandsManual.md | 1 +
.../hadoop/security/token/TestDtUtilShell.java | 44 ++++++++++++++++++++
4 files changed, 107 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
index f154f2d..5f74f8b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtFileOperations.java
@@ -89,7 +89,7 @@ public final class DtFileOperations {
/** Add the service prefix for a local filesystem. */
private static Path fileToPath(File f) {
- return new Path("file:" + f.getAbsolutePath());
+ return new Path(f.toURI().toString());
}
/** Write out a Credentials object as a local file.
@@ -294,4 +294,30 @@ public final class DtFileOperations {
}
doFormattedWrite(tokenFile, fileFormat, creds, conf);
}
+
+ /** Import a token from a base64 encoding into the local filesystem.
+ * @param tokenFile A local File object.
+ * @param fileFormat A string equal to FORMAT_PB or FORMAT_JAVA, for output.
+ * @param alias overwrite Service field of fetched token with this text.
+ * @param base64 urlString Encoding of the token to import.
+ * @param conf Configuration object passed along.
+ * @throws IOException Error to import the token into the file.
+ */
+ public static void importTokenFile(File tokenFile, String fileFormat,
+ Text alias, String base64, Configuration conf)
+ throws IOException {
+
+ Credentials creds = tokenFile.exists() ?
+ Credentials.readTokenStorageFile(tokenFile, conf) : new Credentials();
+
+ Token<TokenIdentifier> token = new Token<>();
+ token.decodeFromUrlString(base64);
+ if (alias != null) {
+ token.setService(alias);
+ }
+ creds.addToken(token.getService(), token);
+ LOG.info("Add token with service {}", token.getService());
+
+ doFormattedWrite(tokenFile, fileFormat, creds, conf);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
index 88db34f..bc2d1b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/DtUtilShell.java
@@ -55,6 +55,7 @@ public class DtUtilShell extends CommandShell {
private static final String CANCEL = "cancel";
private static final String REMOVE = "remove";
private static final String RENEW = "renew";
+ private static final String IMPORT = "import";
private static final String RENEWER = "-renewer";
private static final String SERVICE = "-service";
private static final String ALIAS = "-alias";
@@ -138,6 +139,8 @@ public class DtUtilShell extends CommandShell {
setSubCommand(new Remove(false));
} else if (command.equals(RENEW)) {
setSubCommand(new Renew());
+ } else if (command.equals(IMPORT)) {
+ setSubCommand(new Import(args[++i]));
}
} else if (args[i].equals(ALIAS)) {
alias = new Text(args[++i]);
@@ -176,11 +179,11 @@ public class DtUtilShell extends CommandShell {
@Override
public String getCommandUsage() {
return String.format(
- "%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n%n",
+ "%n%s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n %s%n%n",
DT_USAGE, (new Print()).getUsage(), (new Get()).getUsage(),
(new Edit()).getUsage(), (new Append()).getUsage(),
(new Remove(true)).getUsage(), (new Remove(false)).getUsage(),
- (new Renew()).getUsage());
+ (new Renew()).getUsage(), (new Import()).getUsage());
}
private class Print extends SubCommand {
@@ -357,6 +360,36 @@ public class DtUtilShell extends CommandShell {
}
}
+ private class Import extends SubCommand {
+ public static final String IMPORT_USAGE =
+ "dtutil import <base64> [-alias <alias>] " +
+ FORMAT_SUBSTRING + " filename";
+
+ private String base64 = null;
+
+ Import() { }
+
+ Import(String arg) {
+ base64 = arg;
+ }
+
+ @Override
+ public boolean validate() {
+ return true;
+ }
+
+ @Override
+ public void execute() throws Exception {
+ DtFileOperations.importTokenFile(
+ firstFile, format, alias, base64, getConf());
+ }
+
+ @Override
+ public String getUsage() {
+ return IMPORT_USAGE;
+ }
+ }
+
public static void main(String[] args) throws Exception {
System.exit(ToolRunner.run(new Configuration(), new DtUtilShell(), args));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index ce904c5..f39a92d 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -172,6 +172,7 @@ For every subcommand that connects to a service, convenience flags are provided
| `remove -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | From each file specified, remove the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
| `cancel -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | Just like `remove`, except the tokens are also cancelled using the service specified in the token object. <br/> *alias* must be specified. |
| `renew -alias` *alias* <br/> `[-format (java|protobuf)]` <br/> *filename* `[` *filename2* `...]` | For each file specified, renew the tokens matching *alias* and write out each file using specified format. <br/> *alias* must be specified. |
+| `import` *base64* <br/> `[-alias` *alias* `]` <br/> *filename* | Import a token from a base64 token. <br/> *alias* will overwrite the service field in the token. |
### `fs`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/44e37b4f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
index 53b0d3d..08554fc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/TestDtUtilShell.java
@@ -57,6 +57,13 @@ public class TestDtUtilShell {
public static Text KIND_GET = new Text("testTokenKindGet");
public static Token<?> MOCK_TOKEN =
new Token(IDENTIFIER, PASSWORD, KIND_GET, SERVICE_GET);
+
+ private static final Text SERVICE_IMPORT =
+ new Text("testTokenServiceImport");
+ private static final Text KIND_IMPORT = new Text("testTokenKindImport");
+ private static final Token<?> IMPORT_TOKEN =
+ new Token(IDENTIFIER, PASSWORD, KIND_IMPORT, SERVICE_IMPORT);
+
static {
try {
defaultConf.set("fs.defaultFS", "file:///");
@@ -73,9 +80,11 @@ public class TestDtUtilShell {
private final Path tokenFile2 = new Path(workDir, "testPrintTokenFile2");
private final Path tokenLegacyFile = new Path(workDir, "testPrintTokenFile3");
private final Path tokenFileGet = new Path(workDir, "testGetTokenFile");
+ private final Path tokenFileImport = new Path(workDir, "testImportTokenFile");
private final String tokenFilename = tokenFile.toString();
private final String tokenFilename2 = tokenFile2.toString();
private final String tokenFilenameGet = tokenFileGet.toString();
+ private final String tokenFilenameImport = tokenFileImport.toString();
private String[] args = null;
private DtUtilShell dt = null;
private int rc = 0;
@@ -283,4 +292,39 @@ public class TestDtUtilShell {
spyCreds.readTokenStorageStream(in);
Mockito.verify(spyCreds, Mockito.never()).readFields(in);
}
+
+ @Test
+ public void testImport() throws Exception {
+ String base64 = IMPORT_TOKEN.encodeToUrlString();
+ args = new String[] {"import", base64, tokenFilenameImport};
+ rc = dt.run(args);
+ assertEquals("test simple import print old exit code", 0, rc);
+
+ args = new String[] {"print", tokenFilenameImport};
+ rc = dt.run(args);
+ assertEquals("test simple import print old exit code", 0, rc);
+ assertTrue("test print after import output:\n" + outContent,
+ outContent.toString().contains(KIND_IMPORT.toString()));
+ assertTrue("test print after import output:\n" + outContent,
+ outContent.toString().contains(SERVICE_IMPORT.toString()));
+ assertTrue("test print after simple import output:\n" + outContent,
+ outContent.toString().contains(base64));
+ }
+
+ @Test
+ public void testImportWithAliasFlag() throws Exception {
+ String base64 = IMPORT_TOKEN.encodeToUrlString();
+ args = new String[] {"import", base64, "-alias", alias,
+ tokenFilenameImport};
+ rc = dt.run(args);
+ assertEquals("test import with alias print old exit code", 0, rc);
+
+ args = new String[] {"print", tokenFilenameImport};
+ rc = dt.run(args);
+ assertEquals("test simple import print old exit code", 0, rc);
+ assertTrue("test print after import output:\n" + outContent,
+ outContent.toString().contains(KIND_IMPORT.toString()));
+ assertTrue("test print after import with alias output:\n" + outContent,
+ outContent.toString().contains(alias));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org