You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2017/05/17 07:41:36 UTC
oozie git commit: review fixes
Repository: oozie
Updated Branches:
refs/heads/oya 0e9e5e380 -> 5dd08b159
review fixes
Change-Id: I262eafa0dc5b83875dc136cd555ea162fe320d54
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5dd08b15
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5dd08b15
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5dd08b15
Branch: refs/heads/oya
Commit: 5dd08b1596efb0293409e6bb4f308aa99c25de21
Parents: 0e9e5e3
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Wed May 17 09:40:15 2017 +0200
Committer: Gezapeti Cseh <ge...@gmail.com>
Committed: Wed May 17 09:41:24 2017 +0200
----------------------------------------------------------------------
.../action/hadoop/CredentialsProvider.java | 2 +-
.../hadoop/CredentialsProviderFactory.java | 3 ++-
.../oozie/action/hadoop/FsELFunctions.java | 1 -
.../action/hadoop/HCatCredentialHelper.java | 19 ++++++++--------
.../oozie/action/hadoop/HbaseCredentials.java | 2 --
.../oozie/action/hadoop/JavaActionExecutor.java | 23 +++++++++++---------
.../oozie/action/hadoop/LauncherHelper.java | 1 -
.../action/hadoop/MapReduceActionExecutor.java | 1 -
.../action/hadoop/SparkActionExecutor.java | 12 ----------
.../oozie/command/wf/SubmitMRXCommand.java | 1 -
.../oozie/service/HadoopAccessorService.java | 17 +++------------
.../java/org/apache/oozie/service/Services.java | 1 -
.../apache/oozie/servlet/CallbackServlet.java | 1 -
.../org/apache/oozie/util/ClasspathUtils.java | 3 ---
.../org/apache/oozie/util/Instrumentation.java | 2 --
core/src/main/resources/oozie-default.xml | 8 -------
.../oozie/action/hadoop/TestJavaMain.java | 2 +-
.../java/org/apache/oozie/test/XTestCase.java | 4 ++--
docs/src/site/twiki/ENG_Building.twiki | 4 ++--
.../oozie/action/hadoop/HdfsOperations.java | 3 +--
.../apache/oozie/action/hadoop/JavaMain.java | 9 ++++++++
.../apache/oozie/action/hadoop/LauncherAM.java | 5 +++--
.../oozie/action/hadoop/LauncherMain.java | 8 ++-----
.../oozie/action/hadoop/LauncherMapper.java | 11 +---------
.../oozie/action/hadoop/MapReduceMain.java | 10 ---------
.../action/hadoop/LauncherAMTestMainClass.java | 2 +-
26 files changed, 51 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java
index 1801b2d..d49da90 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProvider.java
@@ -26,7 +26,7 @@ public interface CredentialsProvider {
/**
* This is the interface for all the Credentials implementation. Any new credential implementation must implement
- * this function. This function should modify the jobconf which will be used further to pass the credentials to the
+ * this function. This function should modify the configuration which will be used further to pass the credentials to the
* tasks while running it. Credentials properties and context is also provided by that user can get all the
* necessary configuration.
*
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
index ddffc79..eb9f79a 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/CredentialsProviderFactory.java
@@ -21,6 +21,7 @@ package org.apache.oozie.action.hadoop;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.security.UserGroupInformation;
@@ -46,7 +47,7 @@ public class CredentialsProviderFactory {
}
private CredentialsProviderFactory() throws Exception {
- providerCache = new HashMap<>();
+ providerCache = new ConcurrentHashMap<>();
for (String function : ConfigurationService.getStrings(CRED_KEY)) {
function = trim(function);
LOG.debug("Creating Credential class for : " + function);
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java
index 210747a..c16f560 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java
@@ -186,7 +186,6 @@ public class FsELFunctions {
* ReachingGlobMaxException thrown when globbed file count exceeds the limit
*/
static class ReachingGlobMaxException extends RuntimeException {
- private static final long serialVersionUID = -3569871817672303526L;
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java
index 4d9d3fa..01bd989 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HCatCredentialHelper.java
@@ -33,7 +33,7 @@ import org.apache.oozie.util.XLog;
/**
* Helper class to handle the HCat credentials
* Performs internally the heavy-lifting of fetching delegation tokens from Hive Metastore, abstracted from the user
- * Token is added to jobConf
+ * Token is added to the launcher configuration
*/
public class HCatCredentialHelper {
@@ -45,8 +45,9 @@ public class HCatCredentialHelper {
private static final String HADOOP_RPC_PROTECTION = "hadoop.rpc.protection";
/**
- * This Function will set the HCat token to jobconf
- * @param launcherJobConf - job conf
+ * This Function will set the HCat token to configuration
+ * @param credentials - the credentials
+ * @param launcherConfig - launcher configuration
* @param principal - principal for HCat server
* @param server - Serevr URI for HCat server
* @throws Exception
@@ -56,17 +57,17 @@ public class HCatCredentialHelper {
try {
client = getHCatClient(launcherConfig, principal, server);
XLog.getLog(getClass()).debug(
- "HCatCredentialHelper: set: User name for which token will be asked from HCat: "
- + launcherConfig.get(USER_NAME));
+ "HCatCredentialHelper: set: User name for which token will be asked from HCat: {0}",
+ launcherConfig.get(USER_NAME));
String tokenStrForm = client.getDelegationToken(launcherConfig.get(USER_NAME), UserGroupInformation
.getLoginUser().getShortUserName());
Token<DelegationTokenIdentifier> hcatToken = new Token<DelegationTokenIdentifier>();
hcatToken.decodeFromUrlString(tokenStrForm);
credentials.addToken(new Text("HCat Token"), hcatToken);
- XLog.getLog(getClass()).debug("Added the HCat token in job conf");
+ XLog.getLog(getClass()).debug("Added the HCat token to launcher configuration");
}
catch (Exception ex) {
- XLog.getLog(getClass()).debug("set Exception" + ex.getMessage());
+ XLog.getLog(getClass()).debug("set Exception {0}", ex.getMessage());
throw ex;
}
finally {
@@ -89,7 +90,7 @@ public class HCatCredentialHelper {
HiveConf hiveConf = null;
HCatClient hiveclient = null;
hiveConf = new HiveConf();
- XLog.getLog(getClass()).debug("getHCatClient: Principal: " + principal + " Server: " + server);
+ XLog.getLog(getClass()).debug("getHCatClient: Principal: {0} Server: {1}", principal, server);
// specified a thrift url
hiveConf.set(HIVE_METASTORE_SASL_ENABLED, "true");
@@ -99,7 +100,7 @@ public class HCatCredentialHelper {
String protection = launcherConfig.get(HADOOP_RPC_PROTECTION,
SaslRpcServer.QualityOfProtection.AUTHENTICATION.name()
.toLowerCase());
- XLog.getLog(getClass()).debug("getHCatClient, setting rpc protection to " + protection);
+ XLog.getLog(getClass()).debug("getHCatClient, setting rpc protection to {0}", protection);
hiveConf.set(HADOOP_RPC_PROTECTION, protection);
hiveclient = HCatClient.create(hiveConf);
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java b/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java
index 327ba83..22b6dc9 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HbaseCredentials.java
@@ -102,6 +102,4 @@ public class HbaseCredentials implements CredentialsProvider {
}
}
}
-
-
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 7a762ab..87deb69 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -153,7 +153,6 @@ public class JavaActionExecutor extends ActionExecutor {
protected XLog LOG = XLog.getLog(getClass());
private static final String JAVA_TMP_DIR_SETTINGS = "-Djava.io.tmpdir=";
- private static final LauncherInputFormatClassLocator launcherInputFormatClassLocator = new LauncherInputFormatClassLocator();
public XConfiguration workflowConf = null;
@@ -174,9 +173,6 @@ public class JavaActionExecutor extends ActionExecutor {
public static List<Class<?>> getCommonLauncherClasses() {
List<Class<?>> classes = new ArrayList<Class<?>>();
classes.add(LauncherMain.class);
- classes.add(launcherInputFormatClassLocator.locateOrGet());
- classes.add(OozieLauncherOutputFormat.class);
- classes.add(OozieLauncherOutputCommitter.class);
classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
classes.add(LauncherAM.class);
classes.add(LauncherAMCallbackNotifier.class);
@@ -970,6 +966,7 @@ public class JavaActionExecutor extends ActionExecutor {
// if user-retry is on, always submit new launcher
boolean isUserRetry = ((WorkflowActionBean)action).isUserRetry();
+ LOG.debug("creating yarnClinet for action {0}", action.getId());
yarnClient = createYarnClient(context, launcherJobConf);
if (alreadyRunning && !isUserRetry) {
@@ -1133,6 +1130,7 @@ public class JavaActionExecutor extends ActionExecutor {
// TODO: OYA: make resources allocated for the AM configurable and choose good defaults (memory MB, vcores)
Resource resource = Resource.newInstance(2048, 1);
appContext.setResource(resource);
+ appContext.setCancelTokensWhenComplete(true);
return appContext;
}
@@ -1184,8 +1182,8 @@ public class JavaActionExecutor extends ActionExecutor {
String credName = entry.getKey();
CredentialsProperties credProps = entry.getValue();
if (credProps != null) {
- CredentialsProviderFactory tokenProviderFactory = CredentialsProviderFactory.getInstance();
- CredentialsProvider tokenProvider = tokenProviderFactory.createCredentialsProvider(credProps.getType());
+ CredentialsProvider tokenProvider = CredentialsProviderFactory.getInstance()
+ .createCredentialsProvider(credProps.getType());
if (tokenProvider != null) {
tokenProvider.updateCredentials(credentials, jobconf, credProps, context);
LOG.debug("Retrieved Credential '" + credName + "' for action " + action.getId());
@@ -1341,8 +1339,9 @@ public class JavaActionExecutor extends ActionExecutor {
/**
* If returns true, it means that we have to add Hadoop MR jars to the classpath.
- * Subclasses should override this method if necessary.
- *
+ * Subclasses should override this method if necessary. By default we don't add
+ * MR jars to the classpath.
+ * @return false by default
*/
protected boolean needToAddMRJars() {
return false;
@@ -1379,7 +1378,7 @@ public class JavaActionExecutor extends ActionExecutor {
}
} catch (Exception ye) {
- LOG.info("Exception occurred while checking Launcher AM status; will try checking action data file instead ", ye);
+ LOG.warn("Exception occurred while checking Launcher AM status; will try checking action data file instead ", ye);
// Fallback to action data file if we can't find the Launcher AM (maybe it got purged)
fallback = true;
}
@@ -1511,7 +1510,11 @@ public class JavaActionExecutor extends ActionExecutor {
}
for(ApplicationId id : LauncherMain.getChildYarnJobs(jobConf, ApplicationsRequestScope.ALL,
action.getStartTime().getTime())){
- yarnClient.killApplication(id);
+ try {
+ yarnClient.killApplication(id);
+ } catch (Exception e) {
+ LOG.warn("Could not kill child of {0}, {1}", action.getExternalId(), id);
+ }
}
context.setExternalStatus(KILLED);
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
index 442f9a3..f80141c 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
@@ -51,7 +51,6 @@ import org.apache.oozie.service.UserGroupInformationService;
import org.apache.oozie.util.IOUtils;
import org.apache.oozie.util.PropertiesUtils;
-// TODO: we're no longer using Launcher Mapper -- give this class a better name
public class LauncherHelper {
public static final String OOZIE_ACTION_YARN_TAG = "oozie.action.yarn.tag";
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
index bbc0e89..2050591 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
@@ -133,7 +133,6 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
@Override
protected Configuration createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) {
Configuration conf = super.createBaseHadoopConf(context, actionXml, loadResources);
- injectConfigClass(conf, actionXml);
return conf;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index b237702..2161af0 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -36,7 +36,6 @@ import java.util.Properties;
public class SparkActionExecutor extends JavaActionExecutor {
public static final String SPARK_MAIN_CLASS_NAME = "org.apache.oozie.action.hadoop.SparkMain";
- public static final String TASK_USER_PRECEDENCE = "mapreduce.task.classpath.user.precedence";
public static final String SPARK_MASTER = "oozie.spark.master";
public static final String SPARK_MODE = "oozie.spark.mode";
public static final String SPARK_OPTS = "oozie.spark.spark-opts";
@@ -101,17 +100,6 @@ public class SparkActionExecutor extends JavaActionExecutor {
}
@Override
- Configuration createLauncherConf(FileSystem actionFs, Context context, WorkflowAction action, Element actionXml,
- Configuration actionConf) throws ActionExecutorException {
-
- Configuration launcherJobConf = super.createLauncherConf(actionFs, context, action, actionXml, actionConf);
- if (launcherJobConf.get("oozie.launcher." + TASK_USER_PRECEDENCE) == null) {
- launcherJobConf.set(TASK_USER_PRECEDENCE, "true");
- }
- return launcherJobConf;
- }
-
- @Override
Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
throws ActionExecutorException {
super.setupLauncherConf(conf, actionXml, appPath, context);
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
index 79f0beb..05e7595 100644
--- a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java
@@ -44,7 +44,6 @@ public class SubmitMRXCommand extends SubmitHttpXCommand {
SKIPPED_CONFS.add(XOozieClient.RM);
SKIPPED_CONFS.add(XOozieClient.NN);
- //DEPRECATE_MAP.put(XOozieClient.RM, "yarn.resourcemanager.address");
DEPRECATE_MAP.put(XOozieClient.NN, XOozieClient.NN_2);
DEPRECATE_MAP.put(WorkflowAppService.HADOOP_USER, "mapreduce.job.user.name");
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
index 45f91a3..9624104 100644
--- a/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
+++ b/core/src/main/java/org/apache/oozie/service/HadoopAccessorService.java
@@ -25,11 +25,9 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -47,7 +45,6 @@ import org.apache.oozie.workflow.lite.LiteWorkflowAppParser;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
@@ -523,10 +520,7 @@ public class HadoopAccessorService implements Service {
});
return jobClient;
}
- catch (InterruptedException ex) {
- throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
- }
- catch (IOException ex) {
+ catch (IOException | InterruptedException ex) {
throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
}
}
@@ -571,9 +565,7 @@ public class HadoopAccessorService implements Service {
}
});
return yarnClient;
- } catch (InterruptedException ex) {
- throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
- } catch (IOException ex) {
+ } catch (IOException | InterruptedException ex) {
throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
}
}
@@ -624,10 +616,7 @@ public class HadoopAccessorService implements Service {
}
});
}
- catch (InterruptedException ex) {
- throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
- }
- catch (IOException ex) {
+ catch (IOException | InterruptedException ex) {
throw new HadoopAccessorException(ErrorCode.E0902, ex.getMessage(), ex);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/service/Services.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/service/Services.java b/core/src/main/java/org/apache/oozie/service/Services.java
index 4e1775e..7f47f88 100644
--- a/core/src/main/java/org/apache/oozie/service/Services.java
+++ b/core/src/main/java/org/apache/oozie/service/Services.java
@@ -304,7 +304,6 @@ public class Services {
setService(entry.getValue().getClass());
}
} catch (RuntimeException rex) {
- rex.printStackTrace();
log.fatal("Runtime Exception during Services Load. Check your list of '{0}' or '{1}'",
CONF_SERVICE_CLASSES, CONF_SERVICE_EXT_CLASSES, rex);
throw new ServiceException(ErrorCode.E0103, rex.getMessage(), rex);
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java b/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java
index 86ba467..7d92ffc 100644
--- a/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/CallbackServlet.java
@@ -40,7 +40,6 @@ import org.apache.oozie.util.PropertiesUtils;
import org.apache.oozie.util.XLog;
public class CallbackServlet extends JsonRestServlet {
- private static final long serialVersionUID = 6439106936153152786L;
private static final String INSTRUMENTATION_NAME = "callback";
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java b/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
index 8533371..5833607 100644
--- a/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
+++ b/core/src/main/java/org/apache/oozie/util/ClasspathUtils.java
@@ -41,9 +41,6 @@ public class ClasspathUtils {
private static boolean usingMiniYarnCluster = false;
private static final List<String> CLASSPATH_ENTRIES = Arrays.asList(
ApplicationConstants.Environment.PWD.$(),
- MRJobConfig.JOB_JAR + Path.SEPARATOR + MRJobConfig.JOB_JAR,
- MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR,
- MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*",
ApplicationConstants.Environment.PWD.$() + Path.SEPARATOR + "*"
);
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/java/org/apache/oozie/util/Instrumentation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/util/Instrumentation.java b/core/src/main/java/org/apache/oozie/util/Instrumentation.java
index 87d2eae..45219a9 100644
--- a/core/src/main/java/org/apache/oozie/util/Instrumentation.java
+++ b/core/src/main/java/org/apache/oozie/util/Instrumentation.java
@@ -32,7 +32,6 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
@@ -696,7 +695,6 @@ public class Instrumentation {
public String toString() {
String E = System.getProperty("line.separator");
StringBuilder sb = new StringBuilder(4096);
-
for (String element : all.keySet()) {
sb.append(element).append(':').append(E);
List<String> groups = new ArrayList<String>(all.get(element).keySet());
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index 61b1d88..4ab7512 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1835,14 +1835,6 @@ will be the requeue interval for the actions which are waiting for a long time w
</property>
<property>
- <name>oozie.action.launcher.mapreduce.input.format.class</name>
- <value>org.apache.oozie.action.hadoop.OozieLauncherInputFormat</value>
- <description>
- Make the Launcher Mapper map-only job's InputFormat class pluggable in order to provide alternative implementations.
- </description>
- </property>
-
- <property>
<name>oozie.action.spark.setup.hadoop.conf.dir</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java
index bb2df43..491e745 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaMain.java
@@ -53,7 +53,7 @@ public class TestJavaMain extends MainTestCase {
// Check Exception handling
try {
JavaMain.main(new String[]{"ex2"});
- } catch(JavaMainException jme) {
+ } catch(JavaMain.JavaMainException jme) {
assertTrue(jme.getCause() instanceof IOException);
assertEquals("throwing exception", jme.getCause().getMessage());
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/core/src/test/java/org/apache/oozie/test/XTestCase.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java
index 7ffabd4..72d646d 100644
--- a/core/src/test/java/org/apache/oozie/test/XTestCase.java
+++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java
@@ -1235,8 +1235,8 @@ public abstract class XTestCase extends TestCase {
final ApplicationId appId = ConverterUtils.toApplicationId(externalId);
final MutableObject<YarnApplicationState> finalState = new MutableObject<YarnApplicationState>();
- JobConf jobConf = Services.get().get(HadoopAccessorService.class).createJobConf(getJobTrackerUri());
- final YarnClient yarnClient = Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), jobConf);
+ Configuration conf = Services.get().get(HadoopAccessorService.class).getCachedConf();
+ final YarnClient yarnClient = Services.get().get(HadoopAccessorService.class).createYarnClient(getTestUser(), conf);
try {
waitFor(60 * 1000, new Predicate() {
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/docs/src/site/twiki/ENG_Building.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki
index 3d23b26..36d77b2 100644
--- a/docs/src/site/twiki/ENG_Building.twiki
+++ b/docs/src/site/twiki/ENG_Building.twiki
@@ -112,9 +112,9 @@ Except for the options marked with =(*)=, the options can be specified in the =t
of the Oozie project. The options marked with =(*)= are used in Maven POMs, thus they don't take effect if
specified in the =test.properties= file (which is loaded by the =XTestCase= class at class initialization time).
-*hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-2) you wish to build Oozie against specifically. It will
+*hadoop.version* =(*)=: indicates the Hadoop version you wish to build Oozie against specifically. It will
substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven.
-For Hadoop 2.x, the default (and minimum) version is 2.6.0.
+The default version is 2.6.0 and that is the minimum supported Hadoop version.
*generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated)
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java
index 8e7b258..874d371 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/HdfsOperations.java
@@ -57,8 +57,7 @@ public class HdfsOperations {
public Void run() throws Exception {
Path finalPath = new Path(actionDir, LauncherAM.ACTION_DATA_SEQUENCE_FILE);
// upload into sequence file
- System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: "
- + new Path(actionDir, LauncherAM.ACTION_DATA_SEQUENCE_FILE).toUri());
+ System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: " + finalPath.toUri());
try (SequenceFile.Writer wr =
seqFileWriterFactory.createSequenceFileWriter(launcherJobConf, finalPath, Text.class, Text.class)) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
index 3b4e0a6..c3e3d3f 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
@@ -60,4 +60,13 @@ public class JavaMain extends LauncherMain {
}
+ /**
+ * Used by JavaMain to wrap a Throwable when an Exception occurs
+ */
+ @SuppressWarnings("serial")
+ static class JavaMainException extends Exception {
+ public JavaMainException(Throwable t) {
+ super(t);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
index 2b55997..9484804 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
@@ -53,6 +53,7 @@ public class LauncherAM {
public static final String OOZIE_ACTION_ID = "oozie.action.id";
public static final String OOZIE_JOB_ID = "oozie.job.id";
public static final String ACTION_PREFIX = "oozie.action.";
+ static final String OOZIE_ACTION_RECOVERY_ID = ACTION_PREFIX + "recovery.id";
public static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = ACTION_PREFIX + "max.output.data";
public static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = ACTION_PREFIX + "main.arg.";
public static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + "count";
@@ -375,7 +376,7 @@ public class LauncherAM {
// Get what actually caused the exception
Throwable cause = ex.getCause();
// If we got a JavaMainException from JavaMain, then we need to unwrap it
- if (JavaMainException.class.isInstance(cause)) {
+ if (JavaMain.JavaMainException.class.isInstance(cause)) {
cause = cause.getCause();
}
if (LauncherMainException.class.isInstance(cause)) {
@@ -429,7 +430,7 @@ public class LauncherAM {
ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
String applicationIdStr = applicationId.toString();
- String recoveryId = Preconditions.checkNotNull(launcherJobConf.get(LauncherMapper.OOZIE_ACTION_RECOVERY_ID),
+ String recoveryId = Preconditions.checkNotNull(launcherJobConf.get(OOZIE_ACTION_RECOVERY_ID),
"RecoveryID should not be null");
Path path = new Path(actionDir, recoveryId);
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
index 0f14ec0..0236e1b 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java
@@ -219,10 +219,8 @@ public abstract class LauncherMain {
for(ApplicationReport appReport : appsList) {
childYarnJobs.add(appReport.getApplicationId());
}
- } catch (IOException ioe) {
+ } catch (YarnException | IOException ioe) {
throw new RuntimeException("Exception occurred while finding child jobs", ioe);
- } catch (YarnException ye) {
- throw new RuntimeException("Exception occurred while finding child jobs", ye);
}
System.out.println("Child yarn jobs are found - " + StringUtils.join(childYarnJobs, ","));
@@ -257,10 +255,8 @@ public abstract class LauncherMain {
}
System.out.println();
}
- } catch (YarnException ye) {
+ } catch (IOException | YarnException ye) {
throw new RuntimeException("Exception occurred while killing child job(s)", ye);
- } catch (IOException ioe) {
- throw new RuntimeException("Exception occurred while killing child job(s)", ioe);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
index eda86f4..412807f 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
@@ -240,7 +240,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
// Get what actually caused the exception
Throwable cause = ex.getCause();
// If we got a JavaMainException from JavaMain, then we need to unwrap it
- if (JavaMainException.class.isInstance(cause)) {
+ if (JavaMain.JavaMainException.class.isInstance(cause)) {
cause = cause.getCause();
}
if (LauncherMainException.class.isInstance(cause)) {
@@ -702,12 +702,3 @@ class LauncherSecurityManager extends SecurityManager {
}
}
-/**
- * Used by JavaMain to wrap a Throwable when an Exception occurs
- */
-@SuppressWarnings("serial")
-class JavaMainException extends Exception {
- public JavaMainException(Throwable t) {
- super(t);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
index 02e3fbb..e0974e8 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
@@ -132,16 +132,6 @@ public class MapReduceMain extends LauncherMain {
return runJob;
}
- @Deprecated
- public static void setStrings(Configuration conf, String key, String[] values) {
- ActionUtils.setStrings(conf, key, values);
- }
-
- @Deprecated
- public static String[] getStrings(Configuration conf, String key) {
- return ActionUtils.getStrings(conf, key);
- }
-
protected JobClient createJobClient(JobConf jobConf) throws IOException {
return new JobClient(jobConf);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/5dd08b15/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java
index 6deca64..718bf64 100644
--- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java
+++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/LauncherAMTestMainClass.java
@@ -35,7 +35,7 @@ public class LauncherAMTestMainClass {
if (args != null && args.length == 1) {
switch (args[0]){
case JAVA_EXCEPTION:
- throw new JavaMainException(new RuntimeException(JAVA_EXCEPTION_MESSAGE));
+ throw new JavaMain.JavaMainException(new RuntimeException(JAVA_EXCEPTION_MESSAGE));
case LAUNCHER_EXCEPTION:
throw new LauncherMainException(LAUNCHER_ERROR_CODE);
case SECURITY_EXCEPTION: