You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2017/01/09 23:21:02 UTC
tez git commit: TEZ-3568. Update SecurityUtils configuration to pick
user provided configuration. Contributed by Harish Jaiprakash.
Repository: tez
Updated Branches:
refs/heads/master ea061bcb4 -> c909f45a7
TEZ-3568. Update SecurityUtils configuration to pick user provided configuration. Contributed by Harish Jaiprakash.
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/c909f45a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/c909f45a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/c909f45a
Branch: refs/heads/master
Commit: c909f45a71672f072ddda4b63eeaf79bac7c7f95
Parents: ea061bc
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jan 9 15:20:16 2017 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jan 9 15:20:16 2017 -0800
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../org/apache/tez/common/TezUtilsInternal.java | 21 ++++++++++++++++++++
.../org/apache/tez/dag/app/DAGAppMaster.java | 2 ++
.../org/apache/tez/runtime/task/TezChild.java | 2 ++
4 files changed, 27 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/c909f45a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e281c92..d49507b 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3568. Update SecurityUtils configuration to pick user provided configuration.
TEZ-3561. Fix wrong tez tarball name in install.md.
TEZ-3565. amConfig should check queuename isEmpty.
TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS
@@ -164,6 +165,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3568. Update SecurityUtils configuration to pick user provided configuration.
TEZ-3559. TEZ_LIB_URIS doesn't work with schemes different than the defaultFS
TEZ-3549. TaskAttemptImpl does not initialize TEZ_TASK_PROGRESS_STUCK_INTERVAL_MS correctly
TEZ-3537. ArrayIndexOutOfBoundsException with empty environment variables/Port YARN-3768 to Tez
http://git-wip-us.apache.org/repos/asf/tez/blob/c909f45a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
----------------------------------------------------------------------
diff --git a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
index b0b8906..7b19293 100644
--- a/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
+++ b/tez-common/src/main/java/org/apache/tez/common/TezUtilsInternal.java
@@ -21,6 +21,8 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.nio.charset.Charset;
import java.util.BitSet;
import java.util.List;
@@ -38,6 +40,7 @@ import com.google.protobuf.TextFormat;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.log4j.Appender;
import org.apache.tez.common.io.NonSyncByteArrayOutputStream;
@@ -48,6 +51,7 @@ import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.hadoop.shim.HadoopShim;
import org.apache.tez.serviceplugins.api.TaskAttemptEndReason;
import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.api.records.DAGProtos;
import org.apache.tez.dag.api.records.DAGProtos.ConfigurationProto;
import org.apache.tez.dag.api.records.DAGProtos.PlanKeyValuePair;
@@ -354,4 +358,21 @@ public class TezUtilsInternal {
hadoopShim.setHadoopCallerContext("tez_app:" + appID.toString());
}
+ @Private
+ public static void setSecurityUtilConfigration(Logger log, Configuration conf) {
+ // Use reflection to invoke SecurityUtil.setConfiguration when available, version 2.6.0 of
+ // hadoop does not support it, it is currently available from 2.9.0.
+ // Remove this when the minimum supported hadoop version has the above method.
+ Class<SecurityUtil> clz = SecurityUtil.class;
+ try {
+ Method method = clz.getMethod("setConfiguration", Configuration.class);
+ method.invoke(null, conf);
+ } catch (NoSuchMethodException e) {
+ // This is not available, so ignore it.
+ } catch (SecurityException | IllegalAccessException | IllegalArgumentException |
+ InvocationTargetException e) {
+ log.warn("Error invoking SecurityUtil.setConfiguration: ", e);
+ throw new TezUncheckedException("Error invoking SecurityUtil.setConfiguration", e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/c909f45a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 7bf5d26..7bda424 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -2430,6 +2430,8 @@ public class DAGAppMaster extends AbstractService {
UserGroupInformation.setConfiguration(conf);
Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
+ TezUtilsInternal.setSecurityUtilConfigration(LOG, conf);
+
DAGAppMaster appMaster =
new DAGAppMaster(applicationAttemptId, containerId, nodeHostString,
Integer.parseInt(nodePortString),
http://git-wip-us.apache.org/repos/asf/tez/blob/c909f45a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
index 022fea3..e8e7391 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/task/TezChild.java
@@ -459,6 +459,8 @@ public class TezChild {
// for each and every task, and reading it back from disk. Also needs to be per vertex.
Limits.setConfiguration(conf);
+ TezUtilsInternal.setSecurityUtilConfigration(LOG, conf);
+
// singleton of ObjectRegistry for this JVM
ObjectRegistryImpl objectRegistry = new ObjectRegistryImpl();