You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2016/12/13 13:19:55 UTC
[24/48] oozie git commit: OOZIE-2592 fix Hive2&Sqoop action when
System.exit() called
OOZIE-2592 fix Hive2&Sqoop action when System.exit() called
Change-Id: Ia5db070d5c5287316393d56c626300c4e7777f34
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5c4a79be
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5c4a79be
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5c4a79be
Branch: refs/heads/oya
Commit: 5c4a79bea4a9ec889c9856c4ef986c846454cc26
Parents: ddbd90f
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Tue Nov 22 15:39:00 2016 +0100
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Tue Nov 22 16:55:39 2016 +0100
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 44 ----------------
.../apache/oozie/action/hadoop/HiveMain.java | 7 ---
.../apache/oozie/action/hadoop/Hive2Main.java | 10 ++--
.../action/hadoop/TestHive2ActionExecutor.java | 55 +++++++++++---------
.../apache/oozie/action/hadoop/SqoopMain.java | 7 ---
5 files changed, 32 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 2ec5266..56226ee 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -37,7 +37,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
@@ -52,8 +51,6 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskLog;
import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
@@ -81,7 +78,6 @@ import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.WorkflowAction;
-import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.command.coord.CoordActionStartXCommand;
import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.HadoopAccessorException;
@@ -333,46 +329,6 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
- // FIXME: is this needed?
- private HashMap<String, List<String>> populateEnvMap(String input) {
- HashMap<String, List<String>> envMaps = new HashMap<String, List<String>>();
- String[] envEntries = input.split(",");
- for (String envEntry : envEntries) {
- String[] envKeyVal = envEntry.split("=");
- String envKey = envKeyVal[0].trim();
- List<String> valList = envMaps.get(envKey);
- if (valList == null) {
- valList = new ArrayList<String>();
- }
- valList.add(envKeyVal[1].trim());
- envMaps.put(envKey, valList);
- }
- return envMaps;
- }
-
- public int extractHeapSizeMB(String input) {
- int ret = 0;
- if(input == null || input.equals(""))
- return ret;
- Matcher m = heapPattern.matcher(input);
- String heapStr = null;
- String heapNum = null;
- // Grabs the last match which takes effect (in case that multiple Xmx options specified)
- while (m.find()) {
- heapStr = m.group(1);
- heapNum = m.group(2);
- }
- if (heapStr != null) {
- // when Xmx specified in Gigabyte
- if(heapStr.endsWith("g") || heapStr.endsWith("G")) {
- ret = Integer.parseInt(heapNum) * 1024;
- } else {
- ret = Integer.parseInt(heapNum);
- }
- }
- return ret;
- }
-
public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf)
throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException {
parseJobXmlAndConfiguration(context, element, appPath, conf, false);
http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index f59750b..242cd6c 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -317,13 +317,6 @@ public class HiveMain extends LauncherMain {
try {
runHive(arguments.toArray(new String[arguments.size()]));
}
- catch (SecurityException ex) {
- if (LauncherSecurityManager.getExitInvoked()) {
- if (LauncherSecurityManager.getExitCode() != 0) {
- throw ex;
- }
- }
- }
finally {
System.out.println("\n<<< Invocation of Hive command completed <<<\n");
writeExternalChildIDs(logFile, HIVE_JOB_IDS_PATTERNS, "Hive");
http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
index ccf2aff..b418b89 100644
--- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
+++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
@@ -240,13 +240,6 @@ public class Hive2Main extends LauncherMain {
try {
runBeeline(arguments.toArray(new String[arguments.size()]), logFile);
}
- catch (SecurityException ex) {
- if (LauncherSecurityManager.getExitInvoked()) {
- if (LauncherSecurityManager.getExitCode() != 0) {
- throw ex;
- }
- }
- }
finally {
System.out.println("\n<<< Invocation of Beeline command completed <<<\n");
writeExternalChildIDs(logFile, HIVE2_JOB_IDS_PATTERNS, "Beeline");
@@ -265,6 +258,9 @@ public class Hive2Main extends LauncherMain {
BeeLine beeLine = new BeeLine();
beeLine.setErrorStream(new PrintStream(new TeeOutputStream(System.err, new FileOutputStream(logFile))));
int status = beeLine.begin(args, null);
+ System.out.println("*** Return value from Beeline is: " + status);
+ System.out.println("*** Current security manager: " + System.getSecurityManager());
+ beeLine.close();
if (status != 0) {
System.exit(status);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
index 030942a..89bcd45 100644
--- a/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
+++ b/sharelib/hive2/src/test/java/org/apache/oozie/action/hadoop/TestHive2ActionExecutor.java
@@ -62,10 +62,9 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase {
setSystemProperty("oozie.service.ActionService.executor.classes", Hive2ActionExecutor.class.getName());
}
- @SuppressWarnings("unchecked")
public void testSetupMethodsForScript() throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
- List<Class> classes = new ArrayList<Class>();
+ List<Class<?>> classes = new ArrayList<>();
classes.add(Hive2Main.class);
assertEquals(classes, ae.getLauncherClasses());
@@ -103,10 +102,9 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase {
assertEquals("--dee", conf.get("oozie.hive2.args.1"));
}
- @SuppressWarnings("unchecked")
public void testSetupMethodsForQuery() throws Exception {
Hive2ActionExecutor ae = new Hive2ActionExecutor();
- List<Class> classes = new ArrayList<Class>();
+ List<Class<?>> classes = new ArrayList<>();
classes.add(Hive2Main.class);
assertEquals(classes, ae.getLauncherClasses());
@@ -245,28 +243,33 @@ public class TestHive2ActionExecutor extends ActionExecutorTestCase {
assertTrue(fs.exists(outputDir));
assertTrue(fs.isDirectory(outputDir));
}
- // Negative testcase with incorrect hive-query.
- {
- String query = getHive2BadScript(inputDir.toString(), outputDir.toString());
- Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
- dataWriter.write(SAMPLE_DATA_TEXT);
- dataWriter.close();
- Context context = createContext(getQueryActionXml(query));
- final String launcherId = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2"));
- waitUntilYarnAppDoneAndAssertSuccess(launcherId);
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
- conf);
- assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
- Hive2ActionExecutor ae = new Hive2ActionExecutor();
- ae.check(context, context.getAction());
- assertTrue(launcherId.equals(context.getAction().getExternalId()));
- assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
- ae.end(context, context.getAction());
- assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
- assertNull(context.getExternalChildIDs());
- }
+ }
+
+ public void testHive2ActionFails() throws Exception {
+ setupHiveServer2();
+ Path inputDir = new Path(getFsTestCaseDir(), INPUT_DIRNAME);
+ Path outputDir = new Path(getFsTestCaseDir(), OUTPUT_DIRNAME);
+ FileSystem fs = getFileSystem();
+
+ String query = getHive2BadScript(inputDir.toString(), outputDir.toString());
+ Writer dataWriter = new OutputStreamWriter(fs.create(new Path(inputDir, DATA_FILENAME)));
+ dataWriter.write(SAMPLE_DATA_TEXT);
+ dataWriter.close();
+ Context context = createContext(getQueryActionXml(query));
+ final String launcherId = submitAction(context, Namespace.getNamespace("uri:oozie:hive2-action:0.2"));
+ waitUntilYarnAppDoneAndAssertSuccess(launcherId);
+ Configuration conf = new XConfiguration();
+ conf.set("user.name", getTestUser());
+ Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ conf);
+ assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
+ Hive2ActionExecutor ae = new Hive2ActionExecutor();
+ ae.check(context, context.getAction());
+ assertTrue(launcherId.equals(context.getAction().getExternalId()));
+ assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
+ ae.end(context, context.getAction());
+ assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
+ assertNull(context.getExternalChildIDs());
}
private String getHive2BadScript(String inputPath, String outputPath) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/5c4a79be/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index 9c6ec6c..92c960f 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -182,13 +182,6 @@ public class SqoopMain extends LauncherMain {
try {
runSqoopJob(sqoopArgs);
}
- catch (SecurityException ex) {
- if (LauncherSecurityManager.getExitInvoked()) {
- if (LauncherSecurityManager.getExitCode() != 0) {
- throw ex;
- }
- }
- }
finally {
System.out.println("\n<<< Invocation of Sqoop command completed <<<\n");
writeExternalChildIDs(logFile, SQOOP_JOB_IDS_PATTERNS, "Sqoop");