You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/11/09 22:48:04 UTC
[1/4] git commit: Fix access to Secure HDFS
Updated Branches:
refs/heads/master 72a601ec3 -> 8af99f235
Fix access to Secure HDFS
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/5f9ed517
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/5f9ed517
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/5f9ed517
Branch: refs/heads/master
Commit: 5f9ed51719c294626087a86c2bcceea7ba2f698b
Parents: 3d4ad84
Author: tgravescs <tg...@yahoo.com>
Authored: Fri Nov 8 08:41:57 2013 -0600
Committer: tgravescs <tg...@yahoo.com>
Committed: Fri Nov 8 08:41:57 2013 -0600
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/5f9ed517/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
index 32901a5..47e958b 100644
--- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala
@@ -132,6 +132,8 @@ class HadoopRDD[K, V](
override def getPartitions: Array[Partition] = {
val jobConf = getJobConf()
+ // add the credentials here as this can be called before SparkContext initialized
+ SparkHadoopUtil.get.addCredentials(jobConf)
val inputFormat = getInputFormat(jobConf)
if (inputFormat.isInstanceOf[Configurable]) {
inputFormat.asInstanceOf[Configurable].setConf(jobConf)
[3/4] git commit: Don't call the doAs if user is unknown or the same
user that is already running
Posted by ma...@apache.org.
Don't call the doAs if user is unknown or the same user that is already running
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/13a19505
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/13a19505
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/13a19505
Branch: refs/heads/master
Commit: 13a19505e4d77c93a279d0273826974b4f4cb0a4
Parents: f95cb04
Author: tgravescs <tg...@yahoo.com>
Authored: Fri Nov 8 12:04:09 2013 -0600
Committer: tgravescs <tg...@yahoo.com>
Committed: Fri Nov 8 12:04:09 2013 -0600
----------------------------------------------------------------------
.../apache/spark/deploy/SparkHadoopUtil.scala | 21 +++++++++++++++-----
.../org/apache/spark/executor/Executor.scala | 5 +----
2 files changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/13a19505/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index c29a301..fc1537f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.spark.SparkException
+import org.apache.spark.{SparkContext, SparkException}
/**
* Contains util methods to interact with Hadoop from Spark.
@@ -34,10 +34,21 @@ class SparkHadoopUtil {
UserGroupInformation.setConfiguration(conf)
def runAsUser(user: String)(func: () => Unit) {
- val ugi = UserGroupInformation.createRemoteUser(user)
- ugi.doAs(new PrivilegedExceptionAction[Unit] {
- def run: Unit = func()
- })
+ // if we are already running as the user intended there is no reason to do the doAs. It
+ // will actually break secure HDFS access as it doesn't fill in the credentials. Also if
+ // the user is UNKNOWN then we shouldn't be creating a remote unknown user
+ // (this is actually the path spark on yarn takes) since SPARK_USER is initialized only
+ // in SparkContext.
+ val currentUser = Option(System.getProperty("user.name")).
+ getOrElse(SparkContext.SPARK_UNKNOWN_USER)
+ if (user != SparkContext.SPARK_UNKNOWN_USER && currentUser != user) {
+ val ugi = UserGroupInformation.createRemoteUser(user)
+ ugi.doAs(new PrivilegedExceptionAction[Unit] {
+ def run: Unit = func()
+ })
+ } else {
+ func()
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/13a19505/core/src/main/scala/org/apache/spark/executor/Executor.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 0a4f10c..5c9bb9d 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -179,10 +179,7 @@ private[spark] class Executor(
}
}
- // the runAsUser breaks secure HDFS access. It needs to add the credentials
- // for the user if running as a user. Comment out for now.
- //override def run(): Unit = SparkHadoopUtil.get.runAsUser(sparkUser) { () =>
- override def run(): Unit = {
+ override def run(): Unit = SparkHadoopUtil.get.runAsUser(sparkUser) { () =>
val startTime = System.currentTimeMillis()
SparkEnv.set(env)
Thread.currentThread.setContextClassLoader(replClassLoader)
[4/4] git commit: Merge pull request #149 from
tgravescs/fixSecureHdfsAccess
Posted by ma...@apache.org.
Merge pull request #149 from tgravescs/fixSecureHdfsAccess
Fix secure hdfs access for spark on yarn
https://github.com/apache/incubator-spark/pull/23 broke secure hdfs access. Not sure if it works with secure hdfs on standalone. Fixing it at least for spark on yarn.
The broadcasting of jobconf change also broke secure hdfs access as it didn't take into account things calling the getPartitions before sparkContext is initialized. The DAGScheduler does this as it tries to getShuffleMapStage.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/8af99f23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/8af99f23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/8af99f23
Branch: refs/heads/master
Commit: 8af99f2356ed19fe43d722ada02f8802cbd46d40
Parents: 72a601e 13a1950
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Sat Nov 9 13:48:00 2013 -0800
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Sat Nov 9 13:48:00 2013 -0800
----------------------------------------------------------------------
.../apache/spark/deploy/SparkHadoopUtil.scala | 21 +++++++++++++++-----
.../scala/org/apache/spark/rdd/HadoopRDD.scala | 2 ++
2 files changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[2/4] git commit: Remove the runAsUser as it breaks secure hdfs access
Posted by ma...@apache.org.
Remove the runAsUser as it breaks secure hdfs access
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/f95cb04e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/f95cb04e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/f95cb04e
Branch: refs/heads/master
Commit: f95cb04e4009f25f6b466173fcab24bd861acb24
Parents: 5f9ed51
Author: tgravescs <tg...@yahoo.com>
Authored: Fri Nov 8 10:07:15 2013 -0600
Committer: tgravescs <tg...@yahoo.com>
Committed: Fri Nov 8 10:07:15 2013 -0600
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/executor/Executor.scala | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/f95cb04e/core/src/main/scala/org/apache/spark/executor/Executor.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/executor/Executor.scala b/core/src/main/scala/org/apache/spark/executor/Executor.scala
index 5c9bb9d..0a4f10c 100644
--- a/core/src/main/scala/org/apache/spark/executor/Executor.scala
+++ b/core/src/main/scala/org/apache/spark/executor/Executor.scala
@@ -179,7 +179,10 @@ private[spark] class Executor(
}
}
- override def run(): Unit = SparkHadoopUtil.get.runAsUser(sparkUser) { () =>
+ // the runAsUser breaks secure HDFS access. It needs to add the credentials
+ // for the user if running as a user. Comment out for now.
+ //override def run(): Unit = SparkHadoopUtil.get.runAsUser(sparkUser) { () =>
+ override def run(): Unit = {
val startTime = System.currentTimeMillis()
SparkEnv.set(env)
Thread.currentThread.setContextClassLoader(replClassLoader)