You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by su...@apache.org on 2020/10/16 13:39:38 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-1286] Added
support to fetch multiple DFS tokens for HDFS federation
This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 268d608 [GOBBLIN-1286] Added support to fetch multiple DFS tokens for HDFS federation
268d608 is described below
commit 268d6084c1ef07e6be44f7499e10af81d9877190
Author: Sandeep Devaiah <Sd...@linkedin.com>
AuthorDate: Fri Oct 16 06:39:29 2020 -0700
[GOBBLIN-1286] Added support to fetch multiple DFS tokens for HDFS federation
Closes #3125 from sandy143651/master
---
.../org/apache/gobblin/util/hadoop/TokenUtils.java | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/hadoop/TokenUtils.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/hadoop/TokenUtils.java
index 8c70a13..fd6463d 100644
--- a/gobblin-utility/src/main/java/org/apache/gobblin/util/hadoop/TokenUtils.java
+++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/hadoop/TokenUtils.java
@@ -321,17 +321,15 @@ public class TokenUtils {
private static void getHdfsToken(Configuration conf, Credentials cred) throws IOException {
FileSystem fs = FileSystem.get(conf);
LOG.info("Getting DFS token from " + fs.getUri());
- Token<?> fsToken = fs.getDelegationToken(getMRTokenRenewerInternal(new JobConf()).toString());
- if (fsToken == null) {
- LOG.error("Failed to fetch DFS token for ");
- throw new IOException("Failed to fetch DFS token.");
+ String renewer = getMRTokenRenewerInternal(new JobConf()).toString();
+ Token<?>[] fsTokens = fs.addDelegationTokens(renewer, cred);
+ for(int i = 0; i < fsTokens.length; i++) {
+ Token<?> token = fsTokens[i];
+ String message =
+ String.format("DFS token fetched from namenode, token kind: %s, token service %s", token.getKind(),
+ token.getService());
+ LOG.info(message);
}
- LOG.info("Created DFS token: " + fsToken.toString());
- LOG.info("Token kind: " + fsToken.getKind());
- LOG.info("Token id: " + Arrays.toString(fsToken.getIdentifier()));
- LOG.info("Token service: " + fsToken.getService());
-
- cred.addToken(fsToken.getService(), fsToken);
}
private static void getOtherNamenodesToken(List<String> otherNamenodes, Configuration conf, Credentials cred)