You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/03/22 11:23:42 UTC
[37/50] [abbrv] oozie git commit: OOZIE-2799 Setting log location for
spark sql on hive (satishsaley)
OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/77817e73
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/77817e73
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/77817e73
Branch: refs/heads/oya
Commit: 77817e73588f665cbe43b0971cd93465d40711fb
Parents: 4483f6d
Author: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Authored: Wed Feb 15 09:18:47 2017 -0800
Committer: Satish Subhashrao Saley <sa...@yahoo-inc.com>
Committed: Wed Feb 15 09:18:47 2017 -0800
----------------------------------------------------------------------
.../action/hadoop/SparkActionExecutor.java | 6 +++
release-log.txt | 1 +
.../apache/oozie/action/hadoop/SparkMain.java | 47 ++++++++++++++++++++
3 files changed, 54 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 8e8d7d3..e2e023e 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -160,4 +160,10 @@ public class SparkActionExecutor extends JavaActionExecutor {
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME);
}
+
+ @Override
+ public String[] getShareLibFilesForActionConf() {
+ return new String[] { "hive-site.xml" };
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 4562524..df586c9 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2799 Setting log location for spark sql on hive (satishsaley)
OOZIE-2792 Hive2 action is not parsing Spark application ID from log file properly when Hive is on Spark (zhengxb2005 via rkanter)
OOZIE-2788 Fix jobs API servlet mapping for EmbeddedOozieServer (abhishekbafna via rkanter)
OOZIE-2778 Copy only jetty.version related server dependencies to distro (asasvari via abhishekbafna)
http://git-wip-us.apache.org/repos/asf/oozie/blob/77817e73/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 3cc6ccb..db1e197 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.lang.reflect.Field;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -41,6 +42,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.log4j.PropertyConfigurator;
import org.apache.spark.deploy.SparkSubmit;
@@ -76,6 +78,8 @@ public class SparkMain extends LauncherMain {
private static final Pattern SPARK_VERSION_1 = Pattern.compile("^1.*");
private static final String SPARK_YARN_JAR = "spark.yarn.jar";
private static final String SPARK_YARN_JARS = "spark.yarn.jars";
+ public static final String HIVE_SITE_CONF = "hive-site.xml";
+
public static void main(String[] args) throws Exception {
run(SparkMain.class, args);
}
@@ -89,6 +93,7 @@ public class SparkMain extends LauncherMain {
setYarnTag(actionConf);
LauncherMainHadoopUtils.killChildYarnJobs(actionConf);
String logFile = setUpSparkLog4J(actionConf);
+ setHiveSite(actionConf);
List<String> sparkArgs = new ArrayList<String>();
sparkArgs.add(MASTER_OPTION);
@@ -225,6 +230,7 @@ public class SparkMain extends LauncherMain {
jarfilter.filter();
jarPath = jarfilter.getApplicationJar();
fixedUris.add(new Path(SPARK_LOG4J_PROPS).toUri());
+ fixedUris.add(new Path(HIVE_SITE_CONF).toUri());
String cachedFiles = StringUtils.join(fixedUris, ",");
if (cachedFiles != null && !cachedFiles.isEmpty()) {
sparkArgs.add("--files");
@@ -519,6 +525,47 @@ public class SparkMain extends LauncherMain {
}
/**
+ * Sets up hive-site.xml
+ *
+ * @param hiveConf
+ * @throws IOException
+ */
+ private void setHiveSite(Configuration hiveConf) throws IOException {
+ // See https://issues.apache.org/jira/browse/HIVE-1411
+ hiveConf.set("datanucleus.plugin.pluginRegistryBundleCheck", "LOG");
+
+ // To ensure that the logs go into container attempt tmp directory
+ // When unset, default is
+ // System.getProperty("java.io.tmpdir") + File.separator +
+ // System.getProperty("user.name")
+ hiveConf.unset("hive.querylog.location");
+ hiveConf.unset("hive.exec.local.scratchdir");
+
+ // Write the action configuration out to hive-site.xml
+ OutputStream os = null;
+ try {
+ os = new FileOutputStream(HIVE_SITE_CONF);
+ hiveConf.writeXml(os);
+ }
+ finally {
+ if (os != null) {
+ os.close();
+ }
+ }
+ // Reset the hiveSiteURL static variable as we just created
+ // hive-site.xml.
+ // If prepare block had a drop partition it would have been initialized
+ // to null.
+ try {
+ Field declaredField = HiveConf.class.getDeclaredField("hiveSiteURL");
+ declaredField.setAccessible(true);
+ declaredField.set(null, HiveConf.class.getClassLoader().getResource("hive-site.xml"));
+ }
+ catch (Throwable ignore) {
+ }
+ }
+
+ /**
* This class is used for filtering out unwanted jars.
*/
static class JarFilter {