You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/02/27 00:15:46 UTC
[03/27] hive git commit: HIVE-13118 : add some logging to LLAP token
related paths (Sergey Shelukhin, reviewed by Siddharth Seth)
HIVE-13118 : add some logging to LLAP token related paths (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1c2d7ba3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1c2d7ba3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1c2d7ba3
Branch: refs/heads/llap
Commit: 1c2d7ba35d5e357e125490e63d9ccc84dbff263c
Parents: 9f61fc6
Author: Sergey Shelukhin <se...@apache.org>
Authored: Tue Feb 23 11:30:22 2016 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Tue Feb 23 11:30:22 2016 -0800
----------------------------------------------------------------------
.../hive/llap/tez/LlapProtocolClientProxy.java | 6 +++++
.../hive/llap/security/LlapTokenIdentifier.java | 2 +-
.../daemon/impl/LlapProtocolServerImpl.java | 3 +++
.../hive/llap/security/LlapSecurityHelper.java | 23 ++++++++++++--------
.../llap/tezplugins/LlapTaskCommunicator.java | 3 +++
.../hive/ql/exec/tez/TezSessionState.java | 4 +++-
6 files changed, 30 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
----------------------------------------------------------------------
diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
index 5b0674a..e8d4148 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java
@@ -472,6 +472,9 @@ public class LlapProtocolClientProxy extends AbstractService {
LlapProtocolBlockingPB proxy = hostProxies.get(hostId);
if (proxy == null) {
if (llapToken == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Creating a client without a token for " + nodeId);
+ }
proxy = new LlapProtocolClientImpl(getConfig(), nodeId.getHostname(),
nodeId.getPort(), retryPolicy, socketFactory);
} else {
@@ -485,6 +488,9 @@ public class LlapProtocolClientProxy extends AbstractService {
SecurityUtil.setTokenService(nodeToken, NetUtils.createSocketAddrForHost(
nodeId.getHostname(), nodeId.getPort()));
ugi.addToken(nodeToken);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Creating a client for " + nodeId + "; the token is " + nodeToken);
+ }
proxy = ugi.doAs(new PrivilegedAction<LlapProtocolBlockingPB>() {
@Override
public LlapProtocolBlockingPB run() {
http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
index c4bee75..23980d0 100644
--- a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java
@@ -71,7 +71,7 @@ public class LlapTokenIdentifier extends AbstractDelegationTokenIdentifier {
@Override
public String toString() {
- return KIND;
+ return KIND + "; " + super.toString();
}
@InterfaceAudience.Private
http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
index c386d77..3a25a66 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java
@@ -238,6 +238,9 @@ public class LlapProtocolServerImpl extends AbstractService
LlapTokenIdentifier llapId = new LlapTokenIdentifier(owner, renewer, realUser);
// TODO: note that the token is not renewable right now and will last for 2 weeks by default.
Token<LlapTokenIdentifier> token = new Token<LlapTokenIdentifier>(llapId, zkSecretManager);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Created LLAP token " + token);
+ }
ByteArrayDataOutput out = ByteStreams.newDataOutput();
try {
token.write(out);
http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
index aa8745d..6d8b2da 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java
@@ -54,6 +54,7 @@ public class LlapSecurityHelper implements LlapTokenProvider {
private ServiceInstanceSet activeInstances;
private final Configuration conf;
private LlapManagementProtocolClientImpl client;
+ private ServiceInstance clientInstance;
private final SocketFactory socketFactory;
private final RetryPolicy retryPolicy;
@@ -87,23 +88,24 @@ public class LlapSecurityHelper implements LlapTokenProvider {
// We could have also added keytab support; right now client must do smth like kinit.
}
Iterator<ServiceInstance> llaps = null;
- ServiceInstance someLlap = null;
- if (client == null) {
+ if (clientInstance == null) {
+ assert client == null;
llaps = getLlapServices(false);
- someLlap = llaps.next();
+ clientInstance = llaps.next();
}
ByteString tokenBytes = null;
boolean hasRefreshed = false;
while (true) {
try {
- tokenBytes = getTokenBytes(someLlap);
+ tokenBytes = getTokenBytes();
break;
} catch (InterruptedException ie) {
throw new RuntimeException(ie);
} catch (IOException ex) {
LOG.error("Cannot get a token, trying a different instance", ex);
client = null;
+ clientInstance = null;
}
if (llaps == null || !llaps.hasNext()) {
if (hasRefreshed) { // Only refresh once.
@@ -112,7 +114,7 @@ public class LlapSecurityHelper implements LlapTokenProvider {
llaps = getLlapServices(true);
hasRefreshed = true;
}
- someLlap = llaps.next();
+ clientInstance = llaps.next();
}
// Stupid protobuf byte-buffer reinvention.
@@ -120,17 +122,20 @@ public class LlapSecurityHelper implements LlapTokenProvider {
DataInputByteBuffer in = new DataInputByteBuffer();
in.reset(tokenBytes.asReadOnlyByteBuffer());
token.readFields(in);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Obtained a LLAP delegation token from " + clientInstance + ": " + token);
+ }
return token;
}
- private ByteString getTokenBytes(
- final ServiceInstance si) throws InterruptedException, IOException {
+ private ByteString getTokenBytes() throws InterruptedException, IOException {
return llapUgi.doAs(new PrivilegedExceptionAction<ByteString>() {
@Override
public ByteString run() throws Exception {
+ assert clientInstance != null;
if (client == null) {
- client = new LlapManagementProtocolClientImpl(
- conf, si.getHost(), si.getManagementPort(), retryPolicy, socketFactory);
+ client = new LlapManagementProtocolClientImpl(conf, clientInstance.getHost(),
+ clientInstance.getManagementPort(), retryPolicy, socketFactory);
}
// Client only connects on the first call, so this has to be done in doAs.
GetTokenRequestProto req = GetTokenRequestProto.newBuilder().build();
http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
----------------------------------------------------------------------
diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
index 76d095a..456121b 100644
--- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
+++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java
@@ -123,6 +123,9 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl {
} else {
this.token = null;
}
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Task communicator with a token " + token);
+ }
Preconditions.checkState((token != null) == UserGroupInformation.isSecurityEnabled());
umbilical = new LlapTaskUmbilicalProtocolImpl(getUmbilical());
http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
index 5f9235b..6675f0d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -276,7 +276,9 @@ public class TezSessionState {
if (UserGroupInformation.isSecurityEnabled()) {
LlapTokenProvider tp = LlapProxy.getOrInitTokenProvider(conf);
Token<LlapTokenIdentifier> token = tp.getDelegationToken();
- LOG.info("Obtained a token: " + token);
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Obtained a LLAP token: " + token);
+ }
llapCredentials = new Credentials();
llapCredentials.addToken(LlapTokenIdentifier.KIND_NAME, token);
}