You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@directory.apache.org by pl...@apache.org on 2017/12/25 02:45:37 UTC
directory-kerby git commit: Update the support of Spark.
Repository: directory-kerby
Updated Branches:
refs/heads/has-project 510eebfb6 -> 2247b985e
Update the support of Spark.
Project: http://git-wip-us.apache.org/repos/asf/directory-kerby/repo
Commit: http://git-wip-us.apache.org/repos/asf/directory-kerby/commit/2247b985
Tree: http://git-wip-us.apache.org/repos/asf/directory-kerby/tree/2247b985
Diff: http://git-wip-us.apache.org/repos/asf/directory-kerby/diff/2247b985
Branch: refs/heads/has-project
Commit: 2247b985ef10f3ad5d6420e40c4e35ae14435ec4
Parents: 510eebf
Author: plusplusjiajia <ji...@intel.com>
Authored: Mon Dec 25 10:42:02 2017 +0800
Committer: plusplusjiajia <ji...@intel.com>
Committed: Mon Dec 25 10:42:02 2017 +0800
----------------------------------------------------------------------
has/supports/spark/README.md | 25 ++++++++++++-
has/supports/spark/spark-v2.0.0.patch | 59 ++++++++++++++++++++++++++++++
has/supports/spark/spark-v2.1.1.patch | 51 --------------------------
3 files changed, 82 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/directory-kerby/blob/2247b985/has/supports/spark/README.md
----------------------------------------------------------------------
diff --git a/has/supports/spark/README.md b/has/supports/spark/README.md
index f08ce50..9ccbbba 100644
--- a/has/supports/spark/README.md
+++ b/has/supports/spark/README.md
@@ -1,7 +1,28 @@
Enable Spark
===============
-## 1. Update spark-env.sh
+## 1. Apply the [patch](https://github.com/apache/directory-kerby/blob/has-project/has/supports/hadoop/hadoop-2.7.2.patch) to hadoop-2.7.2 source code
+```
+git apply hadoop-2.7.2.patch
+```
+
+## 2. Install
+```
+mvn clean install -DskipTests
+```
+
+
+## 3. Apply the patch to spark-v2.0.0 source code
+```
+git apply spark-v2.0.0.patch
+```
+
+## 4. Build
+```
+./build/mvn -Pyarn -Phadoop-2.7 -Dhadoop.version=2.7.2 -DskipTests clean package
+```
+
+## 5. Update spark-env.sh
```
SPARK_HISTORY_OPTS=-Dspark.history.kerberos.enabled=true \
-Dspark.history.kerberos.principal=<sp...@HADOOP.COM> \
@@ -10,7 +31,7 @@ SPARK_HISTORY_OPTS=-Dspark.history.kerberos.enabled=true \
> Note "_HOST" should be replaced with the specific hostname.
-## 2. Spark-submit job
+## 6. Spark-submit job
> YARN mode supported only
```
/bin/spark-submit \
http://git-wip-us.apache.org/repos/asf/directory-kerby/blob/2247b985/has/supports/spark/spark-v2.0.0.patch
----------------------------------------------------------------------
diff --git a/has/supports/spark/spark-v2.0.0.patch b/has/supports/spark/spark-v2.0.0.patch
new file mode 100644
index 0000000..9e560a2
--- /dev/null
+++ b/has/supports/spark/spark-v2.0.0.patch
@@ -0,0 +1,59 @@
+diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+index 9feafc9..5501721 100644
+--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
++++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+@@ -549,7 +549,9 @@ object SparkSubmit {
+
+ // assure a keytab is available from any place in a JVM
+ if (clusterManager == YARN || clusterManager == LOCAL) {
+- if (args.principal != null) {
++ if (args.useHas) {
++ UserGroupInformation.loginUserFromHas()
++ } else if (args.principal != null) {
+ require(args.keytab != null, "Keytab must be specified when principal is specified")
+ if (!new File(args.keytab).exists()) {
+ throw new SparkException(s"Keytab file: ${args.keytab} does not exist")
+diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+index f1761e7..835f4c1 100644
+--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
++++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+@@ -78,6 +78,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
+ var submissionToRequestStatusFor: String = null
+ var useRest: Boolean = true // used internally
+
++ var useHas: Boolean = false
++
+ /** Default properties present in the currently defined defaults file. */
+ lazy val defaultSparkProperties: HashMap[String, String] = {
+ val defaultProperties = new HashMap[String, String]()
+@@ -435,6 +437,9 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
+ case VERSION =>
+ SparkSubmit.printVersionAndExit()
+
++ case USE_HAS =>
++ useHas = true
++
+ case USAGE_ERROR =>
+ printUsageAndExit(1)
+
+diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
+index 6767cc5..ec42de5 100644
+--- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
++++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
+@@ -76,6 +76,8 @@ class SparkSubmitOptionParser {
+ protected final String PRINCIPAL = "--principal";
+ protected final String QUEUE = "--queue";
+
++ protected final String USE_HAS = "--use-has";
++
+ /**
+ * This is the canonical list of spark-submit options. Each entry in the array contains the
+ * different aliases for the same option; the first element of each entry is the "official"
+@@ -115,6 +117,7 @@ class SparkSubmitOptionParser {
+ { REPOSITORIES },
+ { STATUS },
+ { TOTAL_EXECUTOR_CORES },
++ { USE_HAS },
+ };
+
+ /**
http://git-wip-us.apache.org/repos/asf/directory-kerby/blob/2247b985/has/supports/spark/spark-v2.1.1.patch
----------------------------------------------------------------------
diff --git a/has/supports/spark/spark-v2.1.1.patch b/has/supports/spark/spark-v2.1.1.patch
deleted file mode 100644
index c7e40b7..0000000
--- a/has/supports/spark/spark-v2.1.1.patch
+++ /dev/null
@@ -1,51 +0,0 @@
-diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
-index 443f1f5..1fc66f0 100644
---- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
-+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
-@@ -553,7 +553,9 @@ object SparkSubmit {
-
- // assure a keytab is available from any place in a JVM
- if (clusterManager == YARN || clusterManager == LOCAL) {
-- if (args.principal != null) {
-+ if (args.useHas) {
-+ UserGroupInformation.loginUserFromHas()
-+ } else if (args.principal != null) {
- require(args.keytab != null, "Keytab must be specified when principal is specified")
- if (!new File(args.keytab).exists()) {
- throw new SparkException(s"Keytab file: ${args.keytab} does not exist")
-diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
-index f1761e7..5e48419 100644
---- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
-+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
-@@ -78,6 +78,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
- var submissionToRequestStatusFor: String = null
- var useRest: Boolean = true // used internally
-
-+ var useHas: Boolean = false
-+
- /** Default properties present in the currently defined defaults file. */
- lazy val defaultSparkProperties: HashMap[String, String] = {
- val defaultProperties = new HashMap[String, String]()
-@@ -438,6 +440,9 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
- case USAGE_ERROR =>
- printUsageAndExit(1)
-
-+ case USE_HAS =>
-+ useHas = true
-+
- case _ =>
- throw new IllegalArgumentException(s"Unexpected argument '$opt'.")
- }
-diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
-index 6767cc5..49a7678 100644
---- a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
-+++ b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitOptionParser.java
-@@ -76,6 +76,8 @@ class SparkSubmitOptionParser {
- protected final String PRINCIPAL = "--principal";
- protected final String QUEUE = "--queue";
-
-+ protected final String USE_HAS = "--use-has";
-+
- /**
- * This is the canonical list of spark-submit options. Each entry in the array contains the
- * different aliases for the same option; the first element of each entry is the "official"