You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/05/08 10:40:43 UTC
[37/37] oozie git commit: Merge branch 'master' into oya
Merge branch 'master' into oya
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/cb6be836
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/cb6be836
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/cb6be836
Branch: refs/heads/oya
Commit: cb6be8369d1bd80c71c3b92707e71e4e88625364
Parents: 09c5389 f669852
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Mon May 8 12:39:50 2017 +0200
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Mon May 8 12:39:50 2017 +0200
----------------------------------------------------------------------
bin/test-patch | 65 +++---
bin/test-patch-11-findbugs-diff | 79 +++++--
bin/test-patch-20-tests | 31 ++-
.../java/org/apache/oozie/cli/OozieCLI.java | 7 +
.../org/apache/oozie/cli/ValidationUtil.java | 30 +++
.../oozie/client/CoordinatorWfAction.java | 48 ++++
.../org/apache/oozie/client/OozieClient.java | 69 ++++++
.../org/apache/oozie/client/rest/JsonTags.java | 6 +
.../apache/oozie/client/rest/RestConstants.java | 10 +
.../apache/oozie/client/rest/TestJsonUtils.java | 21 +-
core/pom.xml | 12 +-
core/src/main/conf/oozie-env.cmd | 2 +-
core/src/main/conf/oozie-env.sh | 28 ---
core/src/main/conf/oozie-log4j.properties | 13 ++
.../main/java/org/apache/oozie/BaseEngine.java | 85 ++++---
.../java/org/apache/oozie/BundleEngine.java | 50 +---
.../org/apache/oozie/CoordinatorEngine.java | 85 +++----
.../apache/oozie/CoordinatorWfActionBean.java | 86 +++++++
.../main/java/org/apache/oozie/DagEngine.java | 83 +------
.../main/java/org/apache/oozie/ErrorCode.java | 1 +
.../oozie/action/hadoop/JavaActionExecutor.java | 14 +-
.../action/hadoop/SqoopActionExecutor.java | 29 ++-
.../oozie/action/ssh/SshActionExecutor.java | 2 +-
.../org/apache/oozie/command/PurgeXCommand.java | 6 +-
.../command/coord/CoordChangeXCommand.java | 4 +-
.../command/coord/CoordSubmitXCommand.java | 25 ++
.../coord/CoordWfActionInfoXCommand.java | 144 ++++++++++++
.../org/apache/oozie/coord/HCatELFunctions.java | 15 +-
.../logic/CoordInputLogicEvaluatorPhaseOne.java | 2 +-
.../apache/oozie/dependency/HCatURIHandler.java | 41 +++-
.../jpa/WorkflowActionGetJPAExecutor.java | 16 +-
.../oozie/service/MemoryLocksService.java | 9 +-
.../org/apache/oozie/service/PurgeService.java | 1 +
.../org/apache/oozie/service/XLogService.java | 12 +-
.../oozie/service/XLogStreamingService.java | 80 ++-----
.../apache/oozie/service/ZKLocksService.java | 53 +++--
.../oozie/service/ZKXLogStreamingService.java | 138 +++--------
.../apache/oozie/servlet/BaseAdminServlet.java | 73 +++++-
.../apache/oozie/servlet/BaseJobServlet.java | 22 ++
.../java/org/apache/oozie/servlet/JVMInfo.java | 2 +-
.../apache/oozie/servlet/V0AdminServlet.java | 16 +-
.../org/apache/oozie/servlet/V0JobServlet.java | 3 +-
.../apache/oozie/servlet/V1AdminServlet.java | 41 ++--
.../org/apache/oozie/servlet/V1JobServlet.java | 4 +-
.../org/apache/oozie/servlet/V2JobServlet.java | 45 ++++
.../org/apache/oozie/sla/SLACalcStatus.java | 65 ++++--
.../apache/oozie/sla/SLACalculatorMemory.java | 192 +++++++--------
.../sla/listener/SLAEmailEventListener.java | 2 +-
.../apache/oozie/sla/service/SLAService.java | 5 +
.../org/apache/oozie/util/AuthUrlClient.java | 3 +-
.../java/org/apache/oozie/util/LogLine.java | 64 +++++
.../util/SimpleTimestampedMessageParser.java | 15 +-
.../oozie/util/TimestampedMessageParser.java | 105 +++++----
.../main/java/org/apache/oozie/util/XLog.java | 12 +
.../org/apache/oozie/util/XLogAuditFilter.java | 16 --
.../apache/oozie/util/XLogAuditStreamer.java | 79 +++++++
.../apache/oozie/util/XLogErrorStreamer.java | 67 ++++++
.../java/org/apache/oozie/util/XLogFilter.java | 152 +++++++++---
.../org/apache/oozie/util/XLogStreamer.java | 117 +++++++++-
.../org/apache/oozie/util/db/SqlStatement.java | 4 +-
.../src/main/resources/jetty-logging.properties | 26 +++
core/src/main/resources/oozie-default.xml | 232 +++++++++++--------
core/src/main/resources/oozie-log4j.properties | 7 +-
.../oozie/TestCoordinatorEngineStreamLog.java | 9 +-
.../action/hadoop/TestFsActionExecutor.java | 14 ++
.../action/hadoop/TestJavaActionExecutor.java | 14 ++
.../hadoop/TestLauncherHCatURIHandler.java | 28 ++-
.../TestScriptLanguageActionExecutor.java | 5 -
.../action/hadoop/TestShellActionExecutor.java | 18 +-
.../org/apache/oozie/client/TestOozieCLI.java | 93 ++++++--
.../command/coord/TestCoordChangeXCommand.java | 40 +++-
.../command/coord/TestCoordSubmitXCommand.java | 30 +++
.../coord/TestCoordWfActionInfoXCommand.java | 174 ++++++++++++++
.../apache/oozie/coord/TestHCatELFunctions.java | 72 +++++-
.../oozie/dependency/TestHCatURIHandler.java | 21 +-
.../oozie/service/TestConfigurationService.java | 7 +-
.../apache/oozie/service/TestHASLAService.java | 3 +
.../service/TestSparkConfigurationService.java | 6 +-
.../oozie/service/TestXLogStreamingService.java | 57 ++++-
.../service/TestZKXLogStreamingService.java | 46 +++-
.../servlet/MockCoordinatorEngineService.java | 55 ++++-
.../apache/oozie/servlet/TestAdminServlet.java | 32 ++-
.../oozie/servlet/TestV1AdminServlet.java | 13 ++
.../apache/oozie/servlet/TestV2JobServlet.java | 189 +++++++++++++++
.../oozie/sla/TestSLACalculatorMemory.java | 22 +-
.../org/apache/oozie/test/MiniHCatServer.java | 30 ++-
.../org/apache/oozie/test/XHCatTestCase.java | 8 +
.../org/apache/oozie/util/TestClassUtils.java | 7 +-
.../apache/oozie/util/TestHCatURIParser.java | 87 +++++++
.../java/org/apache/oozie/util/TestIOUtils.java | 17 +-
.../org/apache/oozie/util/TestLogStreamer.java | 47 +++-
.../TestSimplifiedTimestampedMessageParser.java | 15 +-
.../util/TestTimestampedMessageParser.java | 9 +-
.../apache/oozie/util/TestWritableUtils.java | 9 +-
.../oozie/util/TestXLogUserFilterParam.java | 34 +--
.../org/apache/oozie/util/TestXmlUtils.java | 8 +-
.../resources/coord-invalid-el-function.xml | 35 +++
.../resources/coord-invalid-output-instance.xml | 58 +++++
.../coord-multiple-input-instance4.xml | 2 +-
.../coord-multiple-input-start-instance2.xml | 2 +-
.../coord-multiple-output-instance4.xml | 2 +-
distro/pom.xml | 6 +
distro/src/main/bin/oozie-setup.sh | 4 -
distro/src/main/bin/oozie-sys.sh | 56 -----
docs/src/site/twiki/AG_Install.twiki | 68 ++----
.../site/twiki/CoordinatorFunctionalSpec.twiki | 6 +-
docs/src/site/twiki/DG_CommandLineTool.twiki | 56 +++--
.../site/twiki/DG_CustomActionExecutor.twiki | 2 +-
docs/src/site/twiki/DG_Examples.twiki | 10 +-
.../src/site/twiki/DG_HCatalogIntegration.twiki | 42 +++-
docs/src/site/twiki/DG_QuickStart.twiki | 10 +-
docs/src/site/twiki/ENG_Building.twiki | 5 +-
docs/src/site/twiki/WebServicesAPI.twiki | 28 +++
.../src/site/twiki/WorkflowFunctionalSpec.twiki | 121 +++-------
docs/src/site/twiki/index.twiki | 2 +-
pom.xml | 16 +-
release-log.txt | 33 +++
.../oozie/server/EmbeddedOozieServer.java | 20 +-
.../ConstraintSecurityHandlerProvider.java | 15 +-
.../oozie/server/TestEmbeddedOozieServer.java | 2 +
.../TestConstraintSecurityHandlerProvider.java | 66 ++++++
.../apache/oozie/action/hadoop/DistcpMain.java | 49 ++--
.../action/hadoop/HCatLauncherURIHandler.java | 6 +-
.../java/org/apache/oozie/util/HCatURI.java | 55 +++--
.../org/apache/oozie/util/HCatURIParser.java | 48 ++++
.../java/org/apache/oozie/util/TestHCatURI.java | 27 ++-
.../apache/oozie/action/hadoop/HiveMain.java | 82 +++----
.../oozie/action/hadoop/LauncherMain.java | 50 ++++
.../src/main/resources/default-log4j.properties | 32 +++
.../oozie/action/hadoop/TestLauncherMain.java | 57 +++++
.../org/apache/oozie/action/hadoop/PigMain.java | 62 +++--
.../oozie/action/hadoop/PigMainWithOldAPI.java | 58 ++---
.../apache/oozie/action/hadoop/SparkMain.java | 56 ++---
.../apache/oozie/action/hadoop/SqoopMain.java | 47 ++--
.../action/hadoop/TestSqoopActionExecutor.java | 106 ++++++++-
webapp/pom.xml | 4 +
webapp/src/main/webapp/oozie-console.js | 4 +-
137 files changed, 3844 insertions(+), 1556 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 2256586,f62c997..048b75c
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@@ -36,11 -35,13 +36,13 @@@ import java.util.Iterator
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Objects;
import java.util.Properties;
import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
+import org.apache.commons.io.IOUtils;
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.primitives.Ints;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileStatus;
@@@ -838,11 -983,12 +840,11 @@@ public class JavaActionExecutor extend
prepareXML);
// Set the launcher Main Class
- LauncherMapperHelper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml));
- LauncherMapperHelper.setupLauncherURIHandlerConf(launcherJobConf);
-
- LauncherMapperHelper.setupMaxOutputData(launcherJobConf, getMaxOutputData(actionConf));
- LauncherMapperHelper.setupMaxExternalStatsSize(launcherJobConf, maxExternalStatsSize);
- LauncherMapperHelper.setupMaxFSGlob(launcherJobConf, maxFSGlobMax);
+ LauncherHelper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml));
+ LauncherHelper.setupLauncherURIHandlerConf(launcherJobConf);
- LauncherHelper.setupMaxOutputData(launcherJobConf, maxActionOutputLen);
++ LauncherHelper.setupMaxOutputData(launcherJobConf, getMaxOutputData(actionConf));
+ LauncherHelper.setupMaxExternalStatsSize(launcherJobConf, maxExternalStatsSize);
+ LauncherHelper.setupMaxFSGlob(launcherJobConf, maxFSGlobMax);
List<Element> list = actionXml.getChildren("arg", ns);
String[] args = new String[list.size()];
@@@ -882,9 -1037,33 +884,19 @@@
}
}
+ @VisibleForTesting
+ protected static int getMaxOutputData(Configuration actionConf) {
+ String userMaxActionOutputLen = actionConf.get("oozie.action.max.output.data");
+ if (userMaxActionOutputLen != null) {
+ Integer i = Ints.tryParse(userMaxActionOutputLen);
+ return i != null ? i : maxActionOutputLen;
+ }
+ return maxActionOutputLen;
+ }
+
- private boolean checkPropertiesToDisableUber(Configuration launcherConf) {
- boolean disable = false;
- if (launcherConf.getBoolean(HADOOP_JOB_CLASSLOADER, false)) {
- disable = true;
- }
- else if (launcherConf.getBoolean(HADOOP_USER_CLASSPATH_FIRST, false)) {
- disable = true;
- }
- return disable;
- }
-
protected void injectCallback(Context context, Configuration conf) {
- String callback = context.getCallbackUrl("$jobStatus");
- if (conf.get("job.end.notification.url") != null) {
- LOG.warn("Overriding the action job end notification URI");
- }
- conf.set("job.end.notification.url", callback);
+ String callback = context.getCallbackUrl(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_JOBSTATUS_TOKEN);
+ conf.set(LauncherAMCallbackNotifier.OOZIE_LAUNCHER_CALLBACK_URL, callback);
}
void injectActionCallback(Context context, Configuration actionConf) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
index 1541a42,22e2874..73cc190
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
@@@ -88,22 -91,30 +90,31 @@@ public class SqoopActionExecutor extend
if (actionXml.getChild("command", ns) != null) {
String command = actionXml.getChild("command", ns).getTextTrim();
StringTokenizer st = new StringTokenizer(command, " ");
- List<String> l = new ArrayList<String>();
while (st.hasMoreTokens()) {
- l.add(st.nextToken());
+ argList.add(st.nextToken());
}
- args = l.toArray(new String[l.size()]);
}
else {
+ @SuppressWarnings("unchecked")
List<Element> eArgs = (List<Element>) actionXml.getChildren("arg", ns);
- args = new String[eArgs.size()];
- for (int i = 0; i < eArgs.size(); i++) {
- args[i] = eArgs.get(i).getTextTrim();
+ for (Element elem : eArgs) {
+ argList.add(elem.getTextTrim());
}
}
+ // If the command is given accidentally as "sqoop import --option"
+ // instead of "import --option" we can make a user's life easier
+ // by removing away the unnecessary "sqoop" token.
+ // However, we do not do this if the command looks like
+ // "sqoop --option", as that's entirely invalid.
+ if (argList.size() > 1 &&
+ argList.get(0).equalsIgnoreCase(SQOOP) &&
+ !argList.get(1).startsWith("-")) {
+ XLog.getLog(getClass()).info(
+ "Found a redundant 'sqoop' prefixing the command. Removing it.");
+ argList.remove(0);
+ }
- setSqoopCommand(actionConf, args);
+ setSqoopCommand(actionConf, argList.toArray(new String[argList.size()]));
return actionConf;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/main/resources/oozie-log4j.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/action/hadoop/TestFsActionExecutor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 3d423b9,d1f53fe..c3ff968
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@@ -2281,6 -2965,20 +2281,20 @@@ public class TestJavaActionExecutor ext
assertNull(context.getAction().getData());
ae.end(context, context.getAction());
- assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
+ assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
}
+
+ public void testMaxOutputDataSetByUser() {
+ Configuration conf = new Configuration(false);
+ conf.set("oozie.action.max.output.data", "10000");
+
+ assertEquals(10000, JavaActionExecutor.getMaxOutputData(conf));
+ }
+
+ public void testInvalidMaxOutputDataSetByUser() {
+ Configuration conf = new Configuration(false);
+ conf.set("oozie.action.max.output.data", "sdasda");
+
+ assertEquals(2048, JavaActionExecutor.getMaxOutputData(conf));
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/command/coord/TestCoordChangeXCommand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
----------------------------------------------------------------------
diff --cc core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
index 2d7b3b4,3c6525d..5957ad6
--- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
@@@ -208,7 -209,10 +209,7 @@@ public class TestConfigurationService e
assertEquals("10", ConfigurationService.get(JPAService.CONF_VALIDATE_DB_CONN_EVICTION_NUM).trim());
assertEquals(2048, ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA));
- assertEquals("http://localhost:8080/oozie?job=", ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL));
+ assertEquals("http://0.0.0.0:11000/oozie?job=", ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL));
- assertEquals(true, ConfigurationService.getBoolean(JavaActionExecutor.CONF_HADOOP_YARN_UBER_MODE));
- assertEquals(false, ConfigurationService.getBoolean(
- "oozie.action.shell.launcher." + JavaActionExecutor.HADOOP_YARN_UBER_MODE));
assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED));
assertEquals(0, ConfigurationService.getStrings("no.defined").length);
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/docs/src/site/twiki/DG_QuickStart.twiki
----------------------------------------------------------------------
diff --cc docs/src/site/twiki/DG_QuickStart.twiki
index 37cb53c,fdc8374..607fd7f
--- a/docs/src/site/twiki/DG_QuickStart.twiki
+++ b/docs/src/site/twiki/DG_QuickStart.twiki
@@@ -45,10 -49,10 +45,10 @@@ available to customise the versions of
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
--Dpig.classifier=<classifier> - default none
+-Dpig.classifier=<classifier> - default h2
-Dsqoop.version=<version> - default 1.4.3
-Dsqoop.classifier=<classifier> - default hadoop100
- -Dtomcat.version=<version> - default 6.0.47
+ -Djetty.version=<version> - default 9.2.19.v20160908
-Dopenjpa.version=<version> - default 2.2.2
-Dxerces.version=<version> - default 2.10.0
-Dcurator.version=<version> - default 2.5.0
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/docs/src/site/twiki/ENG_Building.twiki
----------------------------------------------------------------------
diff --cc docs/src/site/twiki/ENG_Building.twiki
index f0bbe71,019f8f1..67540dd
--- a/docs/src/site/twiki/ENG_Building.twiki
+++ b/docs/src/site/twiki/ENG_Building.twiki
@@@ -216,10 -219,10 +215,10 @@@ the versions of the dependencies
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
--Dpig.classifier=<classifier> - default none
+-Dpig.classifier=<classifier> - default h2
-Dsqoop.version=<version> - default 1.4.3
-Dsqoop.classifier=<classifier> - default hadoop100
- -Dtomcat.version=<version> - default 6.0.47
+ -jetty.version=<version> - default 9.2.19.v20160908
-Dopenjpa.version=<version> - default 2.2.2
-Dxerces.version=<version> - default 2.10.0
-Dcurator.version=<version> - default 2.5.0
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/release-log.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --cc sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index 58051d1,9a411ac..5ec509c
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@@ -27,12 -27,10 +27,13 @@@ import java.io.IOException
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
+ import java.net.URL;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
----------------------------------------------------------------------
diff --cc sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
index 41dc9f0,3dfd606..f3f729f
--- a/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
+++ b/sharelib/sqoop/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
@@@ -164,15 -197,68 +187,57 @@@ public class TestSqoopActionExecutor ex
conn.close();
}
+ /**
+ * Tests a bad command of 'sqoop --username ...' style.
+ * Test asserts that the job will fail.
+ */
+ public void testSqoopActionWithBadCommand() throws Exception {
+ runSqoopActionWithBadCommand(getBadCommandActionXml());
+ }
+
+ private void runSqoopActionWithBadCommand(String actionXml) throws Exception {
+ createDB();
+
- Context context = createContext(actionXml);
- final RunningJob launcherJob = submitAction(context);
- String launcherId = context.getAction().getExternalId();
- waitFor(120 * 1000, new Predicate() {
- public boolean evaluate() throws Exception {
- return launcherJob.isComplete();
- }
- });
- assertTrue(launcherJob.isSuccessful());
- Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
++ Context context = createContext(getActionXml());
++ final String launcherId = submitAction(context);
++ waitUntilYarnAppDoneAndAssertSuccess(launcherId);
++
++ Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(),
+ context.getProtoActionConf());
- assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
++ assertFalse(LauncherHelper.hasIdSwap(actionData));
+
+ SqoopActionExecutor ae = new SqoopActionExecutor();
+ ae.check(context, context.getAction());
+ assertTrue(launcherId.equals(context.getAction().getExternalId()));
+ assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
+ ae.end(context, context.getAction());
+ assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
+ }
+
+ /**
+ * Tests a normal command of 'import --username ...'.
+ */
public void testSqoopAction() throws Exception {
+ runSqoopAction(getActionXml());
+ }
+
+ /**
+ * Tests a redundant command of 'sqoop import --username ...'.
+ * The test guarantees a success, since the redundant 'sqoop' must get removed.
+ */
+ public void testSqoopActionWithRedundantPrefix() throws Exception {
+ runSqoopAction(getRedundantCommandActionXml());
+ }
+
+ private void runSqoopAction(String actionXml) throws Exception {
createDB();
- Context context = createContext(actionXml);
- final RunningJob launcherJob = submitAction(context);
- String launcherId = context.getAction().getExternalId();
- waitFor(120 * 1000, new Predicate() {
- public boolean evaluate() throws Exception {
- return launcherJob.isComplete();
- }
- });
- assertTrue(launcherJob.isSuccessful());
- Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ Context context = createContext(getActionXml());
+ final String launcherId = submitAction(context);
+ waitUntilYarnAppDoneAndAssertSuccess(launcherId);
+ Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(),
context.getProtoActionConf());
- assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
+ assertFalse(LauncherHelper.hasIdSwap(actionData));
SqoopActionExecutor ae = new SqoopActionExecutor();
ae.check(context, context.getAction());
@@@ -222,15 -314,44 +287,38 @@@
assertTrue(hadoopCounters.isEmpty());
}
- public void testSqoopActionFreeFormQuery() throws Exception {
+ /**
+ * Runs a job with arg-style command of 'sqoop --username ...' form that's invalid.
+ * The test ensures it fails.
+ */
+ public void testSqoopActionWithBadRedundantArgsAndFreeFormQuery() throws Exception {
+ runSqoopActionWithBadCommand(getBadArgsActionXml());
+ }
+
+ /**
+ * Runs a job with the arg-style command of 'sqoop import --username ...'.
+ * The test guarantees that the redundant 'sqoop' is auto-removed (job passes).
+ */
+ public void testSqoopActionWithRedundantArgsAndFreeFormQuery() throws Exception {
+ runSqoopActionFreeFormQuery(getArgsActionXmlFreeFromQuery(true));
+ }
+
+ /**
+ * Runs a job with the normal arg-style command of 'import --username ...'.
+ */
+ public void testSqoopActionWithArgsAndFreeFormQuery() throws Exception {
+ runSqoopActionFreeFormQuery(getArgsActionXmlFreeFromQuery(false));
+ }
+
+ private void runSqoopActionFreeFormQuery(String actionXml) throws Exception {
createDB();
- Context context = createContext(getActionXmlFreeFromQuery());
+ Context context = createContext(actionXml);
- final RunningJob launcherJob = submitAction(context);
- String launcherId = context.getAction().getExternalId();
- waitFor(120 * 1000, new Predicate() {
- public boolean evaluate() throws Exception {
- return launcherJob.isComplete();
- }
- });
- assertTrue(launcherJob.isSuccessful());
- Map<String, String> actionData = LauncherMapperHelper.getActionData(getFileSystem(), context.getActionDir(),
+ final String launcherId = submitAction(context);
+ waitUntilYarnAppDoneAndAssertSuccess(launcherId);
+ Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(),
context.getProtoActionConf());
- assertFalse(LauncherMapperHelper.hasIdSwap(actionData));
+ assertFalse(LauncherHelper.hasIdSwap(actionData));
SqoopActionExecutor ae = new SqoopActionExecutor();
ae.check(context, context.getAction());
http://git-wip-us.apache.org/repos/asf/oozie/blob/cb6be836/webapp/pom.xml
----------------------------------------------------------------------
diff --cc webapp/pom.xml
index 36cde07,e4fdfb7..4dc0c30
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@@ -37,9 -37,13 +37,13 @@@
<artifactId>oozie-client</artifactId>
<exclusions>
<exclusion>
- <groupId>org.apache.oozie</groupId>
- <artifactId>oozie-hadoop-auth</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ </exclusion>
</exclusions>
</dependency>