You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/03/22 11:23:12 UTC
[07/50] [abbrv] oozie git commit: Addressing code review comments by
rkanter
Addressing code review comments by rkanter
Change-Id: I9ee84d850ea16b9060996d5ccfe62c9f237f2e7a
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/8c4a8565
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/8c4a8565
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/8c4a8565
Branch: refs/heads/oya
Commit: 8c4a8565090ae5530f7d24c54160eac05ae5e9d0
Parents: 7060f14
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Thu Jan 19 14:07:23 2017 +0100
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Thu Jan 19 14:07:29 2017 +0100
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 8 ++---
.../oozie/service/HadoopAccessorService.java | 11 +++++-
.../org/apache/oozie/client/TestOozieCLI.java | 4 +--
.../service/TestHadoopAccessorService.java | 2 --
distro/src/main/bin/addtowar.sh | 2 +-
docs/src/site/twiki/DG_QuickStart.twiki | 4 +--
docs/src/site/twiki/ENG_Building.twiki | 4 +--
findbugs-filter.xml | 3 ++
pom.xml | 26 --------------
.../apache/oozie/action/hadoop/Hive2Main.java | 2 --
.../oozie/action/hadoop/HdfsOperations.java | 38 ++++++++------------
.../apache/oozie/action/hadoop/LauncherAM.java | 30 +++++++---------
.../oozie/action/hadoop/MapReduceMain.java | 6 +---
.../oozie/action/hadoop/TestLauncherAM.java | 30 +++++++---------
sharelib/spark/pom.xml | 21 +----------
15 files changed, 66 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index eb59ff4..9114da8 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -1503,17 +1503,15 @@ public class JavaActionExecutor extends ActionExecutor {
context.setExternalStatus(KILLED);
context.setExecutionData(KILLED, null);
} catch (Exception ex) {
- LOG.error("Error: ", ex);
+ LOG.error("Error when killing YARN application", ex);
throw convertException(ex);
} finally {
try {
FileSystem actionFs = context.getAppFileSystem();
cleanUpActionDir(actionFs, context);
- if (yarnClient != null) {
- yarnClient.close();
- }
+ Closeables.closeQuietly(yarnClient);
} catch (Exception ex) {
- LOG.error("Error: ", ex);
+ LOG.error("Error when cleaning up action dir", ex);
throw convertException(ex);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
index 3a36588..7713196 100644
--- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
@@ -532,12 +532,21 @@ public class HadoopAccessorService implements Service {
}
}
+ /**
+ * Return a JobClient created with the provided user/group.
+ *
+ *
+ * @param conf Configuration with all necessary information to create the
+ * JobClient.
+ * @return JobClient created with the provided user/group.
+ * @throws HadoopAccessorException if the client could not be created.
+ */
public JobClient createJobClient(String user, Configuration conf) throws HadoopAccessorException {
return createJobClient(user, new JobConf(conf));
}
/**
- * Return a YarnClient created with the provided user and configuration.
+ * Return a YarnClient created with the provided user and configuration. The caller is responsible for closing it when done.
*
* @param user The username to impersonate
* @param conf The conf
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
index 33f602e..ceac0eb 100644
--- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
+++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java
@@ -134,8 +134,8 @@ public class TestOozieCLI extends DagServletTestCase {
String path = getTestCaseDir() + "/" + getName() + ".properties";
Properties props = new Properties();
props.setProperty(OozieClient.USER_NAME, getTestUser());
- props.setProperty(XOozieClient.NN, "localhost:9000");
- props.setProperty(XOozieClient.RM, "localhost:9001");
+ props.setProperty(XOozieClient.NN, "localhost:8020");
+ props.setProperty(XOozieClient.RM, "localhost:8032");
props.setProperty("oozie.libpath", appPath);
props.setProperty("mapred.output.dir", appPath);
props.setProperty("a", "A");
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
index e671b7a..1e80abe 100644
--- a/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestHadoopAccessorService.java
@@ -20,13 +20,11 @@ package org.apache.oozie.service;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.security.authorize.*;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.oozie.test.XFsTestCase;
import org.apache.hadoop.mapred.JobConf;
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/distro/src/main/bin/addtowar.sh
----------------------------------------------------------------------
diff --git a/distro/src/main/bin/addtowar.sh b/distro/src/main/bin/addtowar.sh
index 9f8d53d..688ba6b 100644
--- a/distro/src/main/bin/addtowar.sh
+++ b/distro/src/main/bin/addtowar.sh
@@ -95,7 +95,7 @@ function checkOption() {
#get the list of hadoop jars that will be injected based on the hadoop version
function getHadoopJars() {
version=$1
- if [[ "${version}" =~ 2.* ]]; then
+ if [[ "${version}" =~ (2|3).* ]]; then
suffix="-[0-9.]*"
#List is separated by ":"
hadoopJars="hadoop-mapreduce-client-core${suffix}.jar:hadoop-mapreduce-client-common${suffix}.jar:hadoop-mapreduce-client-jobclient${suffix}.jar:hadoop-mapreduce-client-app${suffix}.jar:hadoop-yarn-common${suffix}.jar:hadoop-yarn-api${suffix}.jar:hadoop-yarn-client${suffix}.jar:hadoop-hdfs${suffix}.jar:hadoop-common${suffix}.jar:hadoop-auth${suffix}.jar:guava*.jar:protobuf-*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar:commons-cli-*.jar:commons-io-*.jar"
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/docs/src/site/twiki/DG_QuickStart.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki
index 2454985..37cb53c 100644
--- a/docs/src/site/twiki/DG_QuickStart.twiki
+++ b/docs/src/site/twiki/DG_QuickStart.twiki
@@ -41,10 +41,10 @@ suitable when same oozie package needs to be used in multiple set-ups with diffe
2. Build with -Puber which will bundle the required libraries in the oozie war. Further, the following options are
available to customise the versions of the dependencies:
--Dhadoop.version=<version> - default 2.4.0
+-Dhadoop.version=<version> - default 2.6.0
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
--Dpig.version=<version> - default 0.12.1
+-Dpig.version=<version> - default 0.16.0
-Dpig.classifier=<classifier> - default h2
-Dsqoop.version=<version> - default 1.4.3
-Dsqoop.classifier=<classifier> - default hadoop100
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/docs/src/site/twiki/ENG_Building.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki
index f3fa863..f0bbe71 100644
--- a/docs/src/site/twiki/ENG_Building.twiki
+++ b/docs/src/site/twiki/ENG_Building.twiki
@@ -114,7 +114,7 @@ specified in the =test.properties= file (which is loaded by the =XTestCase= clas
*hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-2) you wish to build Oozie against specifically. It will
substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven.
-For Hadoop 2.x, the default (and minimum) version is 2.4.0.
+For Hadoop 2.x, the default (and minimum) version is 2.6.0.
*generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated)
@@ -216,7 +216,7 @@ the versions of the dependencies:
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
--Dpig.classifier=<classifier> - default none
+-Dpig.classifier=<classifier> - default h2
-Dsqoop.version=<version> - default 1.4.3
-Dsqoop.classifier=<classifier> - default hadoop100
-Dtomcat.version=<version> - default 6.0.47
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/findbugs-filter.xml
----------------------------------------------------------------------
diff --git a/findbugs-filter.xml b/findbugs-filter.xml
index 53c8cb9..03ee4d1 100644
--- a/findbugs-filter.xml
+++ b/findbugs-filter.xml
@@ -16,14 +16,17 @@
limitations under the License.
-->
<FindBugsFilter>
+ <!-- excluded because eventServie is static and it cannot be simply rewritten -->
<Match>
<Class name="org.apache.oozie.command.XCommand" />
<Field name="eventService" />
<Bug pattern="MS_CANNOT_BE_FINAL" />
</Match>
+ <!-- excluded due to the complicated usage of keySet()/get() methods -->
<Match>
<Class name="org.apache.oozie.util.Instrumentation" />
+ <Method name="toString" />
<Bug pattern="WMI_WRONG_MAP_ITERATOR" />
</Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 81fdf79..24dbd4f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1946,32 +1946,6 @@
</properties>
</profile>
<profile>
- <id>wfgen</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <modules>
- <module>workflowgenerator</module>
- </modules>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <fork>true</fork>
- <source>1.6</source>
- <target>1.6</target>
- <compilerArguments>
- <Xlint/>
- <Xmaxwarns>9999</Xmaxwarns>
- </compilerArguments>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
<id>loginServerExample</id>
<activation>
<activeByDefault>false</activeByDefault>
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
----------------------------------------------------------------------
diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
index 6440d83..9e2ae10 100644
--- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
+++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java
@@ -261,8 +261,6 @@ public class Hive2Main extends LauncherMain {
BeeLine beeLine = new BeeLine();
beeLine.setErrorStream(new PrintStream(new TeeOutputStream(System.err, new FileOutputStream(logFile))));
int status = beeLine.begin(args, null);
- System.out.println("*** Return value from Beeline is: " + status);
- System.out.println("*** Current security manager: " + System.getSecurityManager());
beeLine.close();
if (status != 0) {
System.exit(status);
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java
index 22d12ec..8e7b258 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java
@@ -60,9 +60,8 @@ public class HdfsOperations {
System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: "
+ new Path(actionDir, LauncherAM.ACTION_DATA_SEQUENCE_FILE).toUri());
- SequenceFile.Writer wr = null;
- try {
- wr = seqFileWriterFactory.createSequenceFileWriter(launcherJobConf, finalPath, Text.class, Text.class);
+ try (SequenceFile.Writer wr =
+ seqFileWriterFactory.createSequenceFileWriter(launcherJobConf, finalPath, Text.class, Text.class)) {
if (wr != null) {
for (Entry<String, String> entry : actionData.entrySet()) {
@@ -71,14 +70,6 @@ public class HdfsOperations {
} else {
throw new IOException("SequenceFile.Writer is null for " + finalPath);
}
- } finally {
- if (wr != null) {
- try {
- wr.close();
- } catch (IOException e) {
- throw e;
- }
- }
}
return null;
@@ -101,10 +92,11 @@ public class HdfsOperations {
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- FileSystem fs = FileSystem.get(path.toUri(), conf);
- java.io.Writer writer = new OutputStreamWriter(fs.create(path), DEFAULT_CHARSET);
- writer.write(contents);
- writer.close();
+ try (FileSystem fs = FileSystem.get(path.toUri(), conf);
+ java.io.Writer writer = new OutputStreamWriter(fs.create(path), DEFAULT_CHARSET)) {
+ writer.write(contents);
+ }
+
return null;
}
});
@@ -114,17 +106,17 @@ public class HdfsOperations {
return ugi.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
- FileSystem fs = FileSystem.get(path.toUri(), conf);
- InputStream is = fs.open(path);
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, DEFAULT_CHARSET));
StringBuilder sb = new StringBuilder();
- String contents;
- while ((contents = reader.readLine()) != null) {
- sb.append(contents);
- }
+ try (FileSystem fs = FileSystem.get(path.toUri(), conf);
+ InputStream is = fs.open(path);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is, DEFAULT_CHARSET))) {
- reader.close();
+ String contents;
+ while ((contents = reader.readLine()) != null) {
+ sb.append(contents);
+ }
+ }
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
index 20f6676..2b55997 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
@@ -48,10 +48,6 @@ import com.google.common.base.Preconditions;
public class LauncherAM {
private static final String OOZIE_ACTION_CONF_XML = "oozie.action.conf.xml";
private static final String OOZIE_LAUNCHER_JOB_ID = "oozie.launcher.job.id";
- public static final String ACTIONOUTPUTTYPE_ID_SWAP = "IdSwap";
- public static final String ACTIONOUTPUTTYPE_OUTPUT = "Output";
- public static final String ACTIONOUTPUTTYPE_STATS = "Stats";
- public static final String ACTIONOUTPUTTYPE_EXT_CHILD_ID = "ExtChildID";
public static final String JAVA_CLASS_PATH = "java.class.path";
public static final String OOZIE_ACTION_ID = "oozie.action.id";
@@ -154,7 +150,7 @@ public class LauncherAM {
public void run() throws Exception {
final ErrorHolder errorHolder = new ErrorHolder();
OozieActionResult actionResult = OozieActionResult.FAILED;
- boolean launcerExecutedProperly = false;
+ boolean launcherExecutedProperly = false;
boolean backgroundAction = false;
try {
@@ -173,9 +169,9 @@ public class LauncherAM {
final String[] mainArgs = getMainArguments(launcherJobConf);
printDebugInfo();
setupMainConfiguration();
- launcerExecutedProperly = runActionMain(mainArgs, errorHolder, ugi);
+ launcherExecutedProperly = runActionMain(mainArgs, errorHolder, ugi);
- if (launcerExecutedProperly) {
+ if (launcherExecutedProperly) {
handleActionData();
if (actionData.get(ACTION_DATA_OUTPUT_PROPS) != null) {
System.out.println();
@@ -201,7 +197,7 @@ public class LauncherAM {
System.err.println("Launcher AM execution failed");
e.printStackTrace(System.out);
e.printStackTrace(System.err);
- launcerExecutedProperly = false;
+ launcherExecutedProperly = false;
if (!errorHolder.isPopulated()) {
errorHolder.setErrorCause(e);
errorHolder.setErrorMessage(e.getMessage());
@@ -211,11 +207,11 @@ public class LauncherAM {
try {
ErrorHolder callbackErrorHolder = callbackHandler.getError();
- if (launcerExecutedProperly) {
+ if (launcherExecutedProperly) {
actionResult = backgroundAction ? OozieActionResult.RUNNING : OozieActionResult.SUCCEEDED;
}
- if (!launcerExecutedProperly) {
+ if (!launcherExecutedProperly) {
updateActionDataWithFailure(errorHolder, actionData);
} else if (callbackErrorHolder != null) { // async error from the callback
actionResult = OozieActionResult.FAILED;
@@ -456,23 +452,23 @@ public class LauncherAM {
private void handleActionData() throws IOException {
// external child IDs
processActionData(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS, null,
- ACTION_DATA_EXTERNAL_CHILD_IDS, -1, ACTIONOUTPUTTYPE_EXT_CHILD_ID);
+ ACTION_DATA_EXTERNAL_CHILD_IDS, -1);
// external stats
processActionData(ACTION_PREFIX + ACTION_DATA_STATS, CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE,
- ACTION_DATA_STATS, Integer.MAX_VALUE, ACTIONOUTPUTTYPE_STATS);
+ ACTION_DATA_STATS, Integer.MAX_VALUE);
// output data
processActionData(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, CONF_OOZIE_ACTION_MAX_OUTPUT_DATA,
- ACTION_DATA_OUTPUT_PROPS, 2048, ACTIONOUTPUTTYPE_OUTPUT);
+ ACTION_DATA_OUTPUT_PROPS, 2048);
// id swap
processActionData(ACTION_PREFIX + ACTION_DATA_NEW_ID, null,
- ACTION_DATA_NEW_ID, -1, ACTIONOUTPUTTYPE_ID_SWAP);
+ ACTION_DATA_NEW_ID, -1);
}
private void processActionData(String propertyName, String maxSizePropertyName, String actionDataPropertyName,
- int maxSizeDefault, String type) throws IOException {
+ int maxSizeDefault) throws IOException {
String propValue = System.getProperty(propertyName);
int maxSize = maxSizeDefault;
@@ -484,7 +480,7 @@ public class LauncherAM {
File actionDataFile = new File(propValue);
if (localFsOperations.fileExists(actionDataFile)) {
actionData.put(actionDataPropertyName, localFsOperations.getLocalFileContentAsString(actionDataFile,
- type, maxSize));
+ actionDataPropertyName, maxSize));
}
}
}
@@ -525,7 +521,7 @@ public class LauncherAM {
File externalChildIDs = new File(externalChildIdsProp);
if (localFsOperations.fileExists(externalChildIDs)) {
actionData.put(LauncherAM.ACTION_DATA_EXTERNAL_CHILD_IDS,
- localFsOperations.getLocalFileContentAsString(externalChildIDs, ACTIONOUTPUTTYPE_EXT_CHILD_ID, -1));
+ localFsOperations.getLocalFileContentAsString(externalChildIDs, ACTION_DATA_EXTERNAL_CHILD_IDS, -1));
}
}
} catch (IOException ioe) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
index 16cf4b1..59227d9 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
@@ -151,11 +151,7 @@ public class MapReduceMain extends LauncherMain {
Class<? extends OozieActionConfigurator> actionConfiguratorKlass = klass.asSubclass(OozieActionConfigurator.class);
OozieActionConfigurator actionConfigurator = actionConfiguratorKlass.newInstance();
actionConfigurator.configure(actionConf);
- } catch (ClassNotFoundException e) {
- throw new OozieActionConfiguratorException("An Exception occured while instantiating the action config class", e);
- } catch (InstantiationException e) {
- throw new OozieActionConfiguratorException("An Exception occured while instantiating the action config class", e);
- } catch (IllegalAccessException e) {
+ } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
throw new OozieActionConfiguratorException("An Exception occured while instantiating the action config class", e);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
index f6af98b..9cdedb7 100644
--- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
+++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
@@ -17,10 +17,6 @@
*/
package org.apache.oozie.action.hadoop;
-import static org.apache.oozie.action.hadoop.LauncherAM.ACTIONOUTPUTTYPE_EXT_CHILD_ID;
-import static org.apache.oozie.action.hadoop.LauncherAM.ACTIONOUTPUTTYPE_ID_SWAP;
-import static org.apache.oozie.action.hadoop.LauncherAM.ACTIONOUTPUTTYPE_OUTPUT;
-import static org.apache.oozie.action.hadoop.LauncherAM.ACTIONOUTPUTTYPE_STATS;
import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_EXTERNAL_CHILD_IDS;
import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_NEW_ID;
import static org.apache.oozie.action.hadoop.LauncherAM.ACTION_DATA_OUTPUT_PROPS;
@@ -388,7 +384,7 @@ public class TestLauncherAM {
@Test
public void testUpdateActionDataFailsWithActionError() throws Exception {
setupActionOutputContents();
- given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_EXT_CHILD_ID), anyInt()))
+ given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_EXTERNAL_CHILD_IDS), anyInt()))
.willThrow(new IOException());
thrown.expect(IOException.class);
@@ -498,17 +494,17 @@ public class TestLauncherAM {
private void setupActionOutputContents() throws IOException {
// output files generated by an action
- given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_EXT_CHILD_ID), anyInt()))
- .willReturn(ACTIONOUTPUTTYPE_EXT_CHILD_ID);
+ given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_EXTERNAL_CHILD_IDS), anyInt()))
+ .willReturn(ACTION_DATA_EXTERNAL_CHILD_IDS);
- given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_ID_SWAP), anyInt()))
- .willReturn(ACTIONOUTPUTTYPE_ID_SWAP);
+ given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_NEW_ID), anyInt()))
+ .willReturn(ACTION_DATA_NEW_ID);
- given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_OUTPUT), anyInt()))
- .willReturn(ACTIONOUTPUTTYPE_OUTPUT);
+ given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_OUTPUT_PROPS), anyInt()))
+ .willReturn(ACTION_DATA_OUTPUT_PROPS);
- given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTIONOUTPUTTYPE_STATS), anyInt()))
- .willReturn(ACTIONOUTPUTTYPE_STATS);
+ given(localFsOperationsMock.getLocalFileContentAsString(any(File.class), eq(ACTION_DATA_STATS), anyInt()))
+ .willReturn(ACTION_DATA_STATS);
}
private void setupArgsForMainClass(final String... args) {
@@ -546,10 +542,10 @@ public class TestLauncherAM {
String output = actionData.get(ACTION_DATA_OUTPUT_PROPS);
String idSwap = actionData.get(ACTION_DATA_NEW_ID);
- assertThat("extChildID output", ACTIONOUTPUTTYPE_EXT_CHILD_ID, equalTo(extChildId));
- assertThat("stats output", ACTIONOUTPUTTYPE_STATS, equalTo(stats));
- assertThat("action output", ACTIONOUTPUTTYPE_OUTPUT, equalTo(output));
- assertThat("idSwap output", ACTIONOUTPUTTYPE_ID_SWAP, equalTo(idSwap));
+ assertThat("extChildID output", ACTION_DATA_EXTERNAL_CHILD_IDS, equalTo(extChildId));
+ assertThat("stats output", ACTION_DATA_STATS, equalTo(stats));
+ assertThat("action output", ACTION_DATA_OUTPUT_PROPS, equalTo(output));
+ assertThat("idSwap output", ACTION_DATA_NEW_ID, equalTo(idSwap));
}
private void assertNoActionOutputData() {
http://git-wip-us.apache.org/repos/asf/oozie/blob/8c4a8565/sharelib/spark/pom.xml
----------------------------------------------------------------------
diff --git a/sharelib/spark/pom.xml b/sharelib/spark/pom.xml
index df17879..dec505b 100644
--- a/sharelib/spark/pom.xml
+++ b/sharelib/spark/pom.xml
@@ -251,7 +251,7 @@
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-yarn_2.10</artifactId>
+ <artifactId>spark-yarn_${spark.scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>compile</scope>
<exclusions>
@@ -374,25 +374,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <configuration>
- <target>
- <!-- needed to include Main class in classpath for mini yarn cluster for unit tests -->
- <echo file="${project.build.directory}/test-classes/mrapp-generated-classpath"
- append="true" message=":${project.build.directory}/classes"/>
- </target>
- </configuration>
- <goals>
- <goal>run</goal>
- </goals>
- <phase>generate-test-resources</phase>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<finalName>partial-sharelib</finalName>