You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2015/03/10 23:07:49 UTC
oozie git commit: OOZIE-1964 Hive Server 2 action doesn't return
Hadoop Job IDs (rkanter)
Repository: oozie
Updated Branches:
refs/heads/master 0c90b8c36 -> 42cebf6e2
OOZIE-1964 Hive Server 2 action doesn't return Hadoop Job IDs (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/42cebf6e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/42cebf6e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/42cebf6e
Branch: refs/heads/master
Commit: 42cebf6e224d07c2bb698a2c10e8c6377898e42b
Parents: 0c90b8c
Author: Robert Kanter <rk...@cloudera.com>
Authored: Tue Mar 10 15:03:53 2015 -0700
Committer: Robert Kanter <rk...@cloudera.com>
Committed: Tue Mar 10 15:03:53 2015 -0700
----------------------------------------------------------------------
.../action/hadoop/Hive2ActionExecutor.java | 16 +++++++++++-
.../oozie/action/hadoop/HiveActionExecutor.java | 9 +------
.../oozie/action/hadoop/JavaActionExecutor.java | 10 ++++++++
.../action/hadoop/SqoopActionExecutor.java | 9 +------
release-log.txt | 1 +
.../apache/oozie/action/hadoop/HiveMain.java | 25 +-----------------
.../apache/oozie/action/hadoop/Hive2Main.java | 27 +++++++++++++++++---
.../action/hadoop/TestHive2ActionExecutor.java | 7 +++++
.../oozie/action/hadoop/LauncherMain.java | 27 +++++++++++++++++++-
.../org/apache/oozie/action/hadoop/PigMain.java | 5 +---
.../apache/oozie/action/hadoop/SqoopMain.java | 14 +---------
webapp/src/main/webapp/oozie-console.js | 5 ++--
12 files changed, 91 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
index 8f86c09..704b762 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
@@ -20,13 +20,20 @@ package org.apache.oozie.action.hadoop;
import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
+import java.io.IOException;
+import java.io.StringReader;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.RunningJob;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.HadoopAccessorException;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
@@ -115,7 +122,14 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor {
@Override
protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
- return false;
+ return true;
+ }
+
+ @Override
+ protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context)
+ throws HadoopAccessorException, JDOMException, IOException, URISyntaxException {
+ super.getActionData(actionFs, runningJob, action, context);
+ readExternalChildIDs(action, context);
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
index 701d1c1..dbb50ba 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
@@ -115,14 +115,7 @@ public class HiveActionExecutor extends ScriptLanguageActionExecutor {
protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context)
throws HadoopAccessorException, JDOMException, IOException, URISyntaxException {
super.getActionData(actionFs, runningJob, action, context);
-
- if (action.getData() != null) {
- // Load stored Hadoop jobs ids and promote them as external child
- // ids on job success
- Properties props = new Properties();
- props.load(new StringReader(action.getData()));
- context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS));
- }
+ readExternalChildIDs(action, context);
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 5f8646d..cf22ed6 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -1436,6 +1436,16 @@ public class JavaActionExecutor extends ActionExecutor {
throws HadoopAccessorException, JDOMException, IOException, URISyntaxException {
}
+ protected final void readExternalChildIDs(WorkflowAction action, Context context) throws IOException {
+ if (action.getData() != null) {
+ // Load stored Hadoop jobs ids and promote them as external child ids
+ // See LauncherMain#writeExternalChildIDs for how they are written
+ Properties props = new Properties();
+ props.load(new StringReader(action.getData()));
+ context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS));
+ }
+ }
+
protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
Element eConf = XmlUtils.parseXml(action.getConf());
Namespace ns = eConf.getNamespace();
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
index af77c33..6813a37 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
@@ -243,14 +243,7 @@ public class SqoopActionExecutor extends JavaActionExecutor {
protected void getActionData(FileSystem actionFs, RunningJob runningJob, WorkflowAction action, Context context)
throws HadoopAccessorException, JDOMException, IOException, URISyntaxException{
super.getActionData(actionFs, runningJob, action, context);
-
- if (action.getData() != null) {
- // Load stored Hadoop jobs ids and promote them as external child
- // ids
- Properties props = new Properties();
- props.load(new StringReader(action.getData()));
- context.setExternalChildIDs((String) props.get(LauncherMain.HADOOP_JOBS));
- }
+ readExternalChildIDs(action, context);
}
@Override
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 6ab5737..9bd3af2 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.2.0 release (trunk - unreleased)
+OOZIE-1964 Hive Server 2 action doesn't return Hadoop Job IDs (rkanter)
OOZIE-2126 SSH action can be too fast for Oozie sometimes (rkanter)
OOZIE-2142 Changing the JT whitelist causes running Workflows to stay RUNNING forever (rkanter)
OOZIE-2164 make master parameterizable in Spark action example (wypoon via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index df9828a..5ea4e1a 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -296,30 +296,7 @@ public class HiveMain extends LauncherMain {
}
finally {
System.out.println("\n<<< Invocation of Hive command completed <<<\n");
- writeExternalChildIDs(logFile);
-
- }
- }
-
- private void writeExternalChildIDs(String logFile) {
- // harvesting and recording Hadoop Job IDs
- try {
- Properties jobIds = getHadoopJobIds(logFile, HIVE_JOB_IDS_PATTERNS);
- File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX
- + LauncherMapper.ACTION_DATA_OUTPUT_PROPS));
- OutputStream os = new FileOutputStream(file);
- try {
- jobIds.store(os, "");
- }
- finally {
- os.close();
- }
- System.out.println(" Hadoop Job IDs executed by Hive: " + jobIds.getProperty(HADOOP_JOBS));
- System.out.println();
- }
- catch (Exception e) {
- System.out.println("WARN: Error getting Hadoop Job IDs executed by Hive");
- e.printStackTrace(System.out);
+ writeExternalChildIDs(logFile, HIVE_JOB_IDS_PATTERNS, "Hive");
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
index 6d4f44a..ca9fcb4 100644
--- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
+++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
@@ -20,18 +20,25 @@ package org.apache.oozie.action.hadoop;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.regex.Pattern;
+import org.apache.commons.io.output.TeeOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hive.beeline.BeeLine;
public class Hive2Main extends LauncherMain {
+ private static final Pattern[] HIVE2_JOB_IDS_PATTERNS = {
+ Pattern.compile("Ended Job = (job_\\S*)")
+ };
private static final Set<String> DISALLOWED_BEELINE_OPTIONS = new HashSet<String>();
static {
@@ -93,6 +100,13 @@ public class Hive2Main extends LauncherMain {
Configuration actionConf = initActionConf();
+ //Logfile to capture job IDs
+ String hadoopJobId = System.getProperty("oozie.launcher.job.id");
+ if (hadoopJobId == null) {
+ throw new RuntimeException("Launcher Hadoop Job ID system property not set");
+ }
+ String logFile = new File("hive2-oozie-" + hadoopJobId + ".log").getAbsolutePath();
+
List<String> arguments = new ArrayList<String>();
String jdbcUrl = actionConf.get(Hive2ActionExecutor.HIVE2_JDBC_URL);
if (jdbcUrl == null) {
@@ -199,7 +213,7 @@ public class Hive2Main extends LauncherMain {
System.out.flush();
try {
- runBeeline(arguments.toArray(new String[arguments.size()]));
+ runBeeline(arguments.toArray(new String[arguments.size()]), logFile);
}
catch (SecurityException ex) {
if (LauncherSecurityManager.getExitInvoked()) {
@@ -210,11 +224,18 @@ public class Hive2Main extends LauncherMain {
}
finally {
System.out.println("\n<<< Invocation of Beeline command completed <<<\n");
+ writeExternalChildIDs(logFile, HIVE2_JOB_IDS_PATTERNS, "Beeline");
}
}
- private void runBeeline(String[] args) throws Exception {
- BeeLine.main(args);
+ private void runBeeline(String[] args, String logFile) throws Exception {
+ // We do this instead of calling BeeLine.main so we can duplicate the error stream for harvesting Hadoop child job IDs
+ BeeLine beeLine = new BeeLine();
+ beeLine.setErrorStream(new PrintStream(new TeeOutputStream(System.err, new FileOutputStream(logFile))));
+ int status = beeLine.begin(args, null);
+ if (status != 0) {
+ System.exit(status);
+ }
}
private static String readStringFromFile(String filePath) throws IOException {
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
index ac39b94..16d0267 100644
--- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
+++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
@@ -25,6 +25,7 @@ import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -174,6 +175,12 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase {
ae.end(context, context.getAction());
assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
+ assertNotNull(context.getAction().getData());
+ Properties outputData = new Properties();
+ outputData.load(new StringReader(context.getAction().getData()));
+ assertTrue(outputData.containsKey(LauncherMain.HADOOP_JOBS));
+ assertEquals(outputData.get(LauncherMain.HADOOP_JOBS), context.getExternalChildIDs());
+
assertTrue(fs.exists(outputDir));
assertTrue(fs.isDirectory(outputDir));
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index 552c862..0860484 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -20,8 +20,10 @@ package org.apache.oozie.action.hadoop;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.io.OutputStream;
import java.io.StringWriter;
import java.util.Collection;
import java.util.Map;
@@ -44,7 +46,7 @@ public abstract class LauncherMain {
main.run(args);
}
- public static Properties getHadoopJobIds(String logFile, Pattern[] patterns) throws IOException {
+ protected static Properties getHadoopJobIds(String logFile, Pattern[] patterns) throws IOException {
Properties props = new Properties();
StringBuffer sb = new StringBuffer(100);
if (!new File(logFile).exists()) {
@@ -76,6 +78,29 @@ public abstract class LauncherMain {
return props;
}
+ protected static void writeExternalChildIDs(String logFile, Pattern[] patterns, String name) {
+ // Harvesting and recording Hadoop Job IDs
+ // See JavaActionExecutor#readExternalChildIDs for how they are read
+ try {
+ Properties jobIds = getHadoopJobIds(logFile, patterns);
+ File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX
+ + LauncherMapper.ACTION_DATA_OUTPUT_PROPS));
+ OutputStream os = new FileOutputStream(file);
+ try {
+ jobIds.store(os, "");
+ }
+ finally {
+ os.close();
+ }
+ System.out.println(" Hadoop Job IDs executed by " + name + ": " + jobIds.getProperty(HADOOP_JOBS));
+ System.out.println();
+ }
+ catch (Exception e) {
+ System.out.println("WARN: Error getting Hadoop Job IDs executed by " + name);
+ e.printStackTrace(System.out);
+ }
+ }
+
protected abstract void run(String[] args) throws Exception;
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index d924018..129022a 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -235,10 +235,7 @@ public class PigMain extends LauncherMain {
// So retrieving hadoop Ids here
File file = new File(System.getProperty(EXTERNAL_CHILD_IDS));
if (!file.exists()) {
- Properties props = getHadoopJobIds(logFile, PIG_JOB_IDS_PATTERNS);
- writeExternalData(props.getProperty(HADOOP_JOBS), file);
- System.out.println(" Hadoop Job IDs executed by Pig: " + props.getProperty(HADOOP_JOBS));
- System.out.println();
+ writeExternalChildIDs(logFile, PIG_JOB_IDS_PATTERNS, "Pig");
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index d672bf3..1ffaf10 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -188,19 +188,7 @@ public class SqoopMain extends LauncherMain {
System.out.println();
// harvesting and recording Hadoop Job IDs
- Properties jobIds = getHadoopJobIds(logFile, SQOOP_JOB_IDS_PATTERNS);
-
- File file = new File(System.getProperty(LauncherMapper.ACTION_PREFIX
- + LauncherMapper.ACTION_DATA_OUTPUT_PROPS));
- OutputStream os = new FileOutputStream(file);
- try {
- jobIds.store(os, "");
- }
- finally {
- os.close();
- }
- System.out.println(" Hadoop Job IDs executed by Sqoop: " + jobIds.getProperty(HADOOP_JOBS));
- System.out.println();
+ writeExternalChildIDs(logFile, SQOOP_JOB_IDS_PATTERNS, "Sqoop");
}
protected void runSqoopJob(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/oozie/blob/42cebf6e/webapp/src/main/webapp/oozie-console.js
----------------------------------------------------------------------
diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js
index e2b37a6..e07680f 100644
--- a/webapp/src/main/webapp/oozie-console.js
+++ b/webapp/src/main/webapp/oozie-console.js
@@ -728,7 +728,7 @@ function jobDetailsPopup(response, request) {
})]
});
- // Tab to show list of child Job URLs for pig action
+ // Tab to show list of child Job URLs
var childJobsItem = {
title : 'Child Job URLs',
autoScroll : true,
@@ -739,7 +739,8 @@ function jobDetailsPopup(response, request) {
width: 540,
items : urlUnit
};
- if (actionStatus.type == "pig" || actionStatus.type == "hive" || actionStatus.type == "map-reduce") {
+ if (actionStatus.type == "pig" || actionStatus.type == "hive" || actionStatus.type == "map-reduce"
+ || actionStatus.type == "hive2" || actionStatus.type == "sqoop") {
var tabPanel = win.items.get(0);
tabPanel.add(childJobsItem);
}