You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2016/09/19 17:07:53 UTC
oozie git commit: OOZIE-2569 Adding yarn-site, core-site,
hdfs-site and mapred-site into spark launcher (abhishekbafna via
rohini)
Repository: oozie
Updated Branches:
refs/heads/master f86107155 -> 0cdce5a16
OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/0cdce5a1
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/0cdce5a1
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/0cdce5a1
Branch: refs/heads/master
Commit: 0cdce5a1645799d01520176c17da6ab7b9739c73
Parents: f861071
Author: Rohini Palaniswamy <ro...@apache.org>
Authored: Mon Sep 19 10:07:57 2016 -0700
Committer: Rohini Palaniswamy <ro...@apache.org>
Committed: Mon Sep 19 10:07:57 2016 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/SparkActionExecutor.java | 6 ++++++
core/src/main/resources/oozie-default.xml | 11 +++++++++++
release-log.txt | 1 +
.../org/apache/oozie/action/hadoop/LauncherMain.java | 14 ++++++++++++++
.../org/apache/oozie/action/hadoop/ShellMain.java | 13 ++-----------
.../org/apache/oozie/action/hadoop/SparkMain.java | 13 +++++++++++++
6 files changed, 47 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 97355fd..15a641b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.HadoopAccessorException;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.SparkConfigurationService;
@@ -49,6 +50,7 @@ public class SparkActionExecutor extends JavaActionExecutor {
public static final String SPARK_CLASS = "oozie.spark.class";
public static final String SPARK_JAR = "oozie.spark.jar";
public static final String MAPRED_CHILD_ENV = "mapred.child.env";
+ private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir";
public SparkActionExecutor() {
super("spark");
@@ -95,6 +97,10 @@ public class SparkActionExecutor extends JavaActionExecutor {
actionConf.set(SPARK_OPTS, sparkOptsSb.toString().trim());
}
+ // Setting if SparkMain should setup hadoop config *-site.xml
+ boolean setupHadoopConf = actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR,
+ ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR));
+ actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, setupHadoopConf);
return actionConf;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 40a5fe5..e71ebe3 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1815,6 +1815,17 @@ will be the requeue interval for the actions which are waiting for a long time w
</property>
<property>
+ <name>oozie.action.spark.setup.hadoop.conf.dir</name>
+ <value>false</value>
+ <description>
+ Oozie action.xml (oozie.action.conf.xml) contains all the hadoop configuration and user provided configurations.
+ This property will allow users to copy Oozie action.xml as hadoop *-site configurations files. The advantage is,
+ user need not to manage these files into spark sharelib. If user wants to manage the hadoop configurations
+ themselves, it should should disable it.
+ </description>
+ </property>
+
+ <property>
<name>oozie.action.shell.setup.hadoop.conf.dir</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 681e5ee..7851e81 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2569 Adding yarn-site, core-site, hdfs-site and mapred-site into spark launcher (abhishekbafna via rohini)
OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini)
OOZIE-2588 Support getting credentials for same cluster hcat when credentials config is empty (satishsaley via rohini)
OOZIE-2538 Update HttpClient versions to close security vulnerabilities (abhishekbafna via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index fb190d6..054f8ea 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
import java.util.Properties;
@@ -43,6 +44,8 @@ public abstract class LauncherMain {
public static final String HADOOP_JOBS = "hadoopJobs";
public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags";
+ protected static String[] HADOOP_SITE_FILES = new String[]
+ {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"};
protected static void run(Class<? extends LauncherMain> klass, String[] args) throws Exception {
LauncherMain main = klass.newInstance();
@@ -247,6 +250,17 @@ public abstract class LauncherMain {
}
}
}
+
+ protected void writeHadoopConfig(String actionXml, File basrDir) throws IOException {
+ File actionXmlFile = new File(actionXml);
+ System.out.println("Copying " + actionXml + " to " + basrDir + "/" + Arrays.toString(HADOOP_SITE_FILES));
+ basrDir.mkdirs();
+ File[] dstFiles = new File[HADOOP_SITE_FILES.length];
+ for (int i = 0; i < dstFiles.length; i++) {
+ dstFiles[i] = new File(basrDir, HADOOP_SITE_FILES[i]);
+ }
+ copyFileMultiplex(actionXmlFile, dstFiles);
+ }
}
class LauncherMainException extends Exception {
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
index c4a6e9b..5e80d00 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
@@ -29,7 +29,6 @@ import java.io.StringReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -50,7 +49,6 @@ public class ShellMain extends LauncherMain {
private static final String HADOOP_CONF_DIR = "HADOOP_CONF_DIR";
private static final String YARN_CONF_DIR = "YARN_CONF_DIR";
- private static String[] HADOOP_SITE_FILES = new String[] {"core-site.xml", "hdfs-site.xml", "mapred-site.xml", "yarn-site.xml"};
private static String LOG4J_PROPERTIES = "log4j.properties";
/**
@@ -88,7 +86,7 @@ public class ShellMain extends LauncherMain {
ProcessBuilder builder = new ProcessBuilder(cmdArray);
Map<String, String> envp = getEnvMap(builder.environment(), actionConf);
- // Getting the Ccurrent working dir and setting it to processbuilder
+ // Getting the Current working dir and setting it to processbuilder
File currDir = new File("dummy").getAbsoluteFile().getParentFile();
System.out.println("Current working dir " + currDir);
builder.directory(currDir);
@@ -140,15 +138,8 @@ public class ShellMain extends LauncherMain {
if (actionConf.getBoolean(CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR, false)) {
String actionXml = envp.get(OOZIE_ACTION_CONF_XML);
if (actionXml != null) {
- File actionXmlFile = new File(actionXml);
File confDir = new File(currDir, "oozie-hadoop-conf-" + System.currentTimeMillis());
- System.out.println("Copying " + actionXml + " to " + confDir + "/" + Arrays.toString(HADOOP_SITE_FILES));
- confDir.mkdirs();
- File[] dstFiles = new File[HADOOP_SITE_FILES.length];
- for (int i = 0; i < dstFiles.length; i++) {
- dstFiles[i] = new File(confDir, HADOOP_SITE_FILES[i]);
- }
- copyFileMultiplex(actionXmlFile, dstFiles);
+ writeHadoopConfig(actionXml, confDir);
if (actionConf.getBoolean(CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR_WRITE_LOG4J_PROPERTIES, true)) {
System.out.println("Writing " + LOG4J_PROPERTIES + " to " + confDir);
writeLoggerProperties(actionConf, confDir);
http://git-wip-us.apache.org/repos/asf/oozie/blob/0cdce5a1/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 3acaef9..38e8e8c 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -48,6 +48,7 @@ public class SparkMain extends LauncherMain {
private static final String DRIVER_CLASSPATH = "spark.driver.extraClassPath=";
private static final String HIVE_SECURITY_TOKEN = "spark.yarn.security.tokens.hive.enabled";
private static final String HBASE_SECURITY_TOKEN = "spark.yarn.security.tokens.hbase.enabled";
+ private static final String CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR = "oozie.action.spark.setup.hadoop.conf.dir";
private static final String PWD = "$PWD" + File.separator + "*";
private static final Pattern[] PYSPARK_DEP_FILE_PATTERN = { Pattern.compile("py4\\S*src.zip"),
Pattern.compile("pyspark.zip") };
@@ -63,6 +64,7 @@ public class SparkMain extends LauncherMain {
protected void run(String[] args) throws Exception {
boolean isPyspark = false;
Configuration actionConf = loadActionConf();
+ prepareHadoopConfig(actionConf);
setYarnTag(actionConf);
LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
@@ -214,6 +216,17 @@ public class SparkMain extends LauncherMain {
}
}
+ private void prepareHadoopConfig(Configuration actionConf) throws IOException {
+ // Copying oozie.action.conf.xml into hadoop configuration *-site files.
+ if (actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, false)) {
+ String actionXml = System.getProperty("oozie.action.conf.xml");
+ if (actionXml != null) {
+ File currentDir = new File(actionXml).getParentFile();
+ writeHadoopConfig(actionXml, currentDir);
+ }
+ }
+ }
+
/**
* SparkActionExecutor sets the SPARK_HOME environment variable to the local directory.
* Spark is looking for the pyspark.zip and py4j-VERSION-src.zip files in the python/lib folder under SPARK_HOME.