You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/07/04 12:30:32 UTC
[1/2] oozie git commit: OOZIE-2918 Delete LauncherMapper and its test
(asasvari via pbacsko)
Repository: oozie
Updated Branches:
refs/heads/master 79ccfdd78 -> c09561ff2
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
deleted file mode 100644
index d0d3629..0000000
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
+++ /dev/null
@@ -1,715 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.security.Permission;
-import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.StringTokenizer;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.OutputCollector;
-import org.apache.hadoop.mapred.Reporter;
-import org.xml.sax.SAXException;
-
-import com.google.common.base.Strings;
-import javax.xml.parsers.ParserConfigurationException;
-
-// TODO: OYA: Delete :)
-public class LauncherMapper<K1, V1, K2, V2> implements Runnable {
-
- static final String CONF_OOZIE_ACTION_MAIN_CLASS = "oozie.launcher.action.main.class";
-
- static final String ACTION_PREFIX = "oozie.action.";
- public static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = ACTION_PREFIX + "max.output.data";
- static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = ACTION_PREFIX + "main.arg.count";
- static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = ACTION_PREFIX + "main.arg.";
- static final String CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE = "oozie.external.stats.max.size";
- static final String OOZIE_ACTION_CONFIG_CLASS = ACTION_PREFIX + "config.class";
- static final String CONF_OOZIE_ACTION_FS_GLOB_MAX = ACTION_PREFIX + "fs.glob.max";
- static final String CONF_OOZIE_NULL_ARGS_ALLOWED = ACTION_PREFIX + "null.args.allowed";
-
- static final String COUNTER_GROUP = "oozie.launcher";
- static final String COUNTER_LAUNCHER_ERROR = "oozie.launcher.error";
-
- static final String OOZIE_JOB_ID = "oozie.job.id";
- static final String OOZIE_ACTION_ID = ACTION_PREFIX + "id";
- static final String OOZIE_ACTION_RECOVERY_ID = ACTION_PREFIX + "recovery.id";
-
- static final String OOZIE_ACTION_DIR_PATH = ACTION_PREFIX + "dir.path";
- static final String ACTION_CONF_XML = "action.xml";
- static final String ACTION_PREPARE_XML = "oozie.action.prepare.xml";
- static final String ACTION_DATA_SEQUENCE_FILE = "action-data.seq"; // COMBO FILE
- static final String ACTION_DATA_EXTERNAL_CHILD_IDS = "externalChildIDs";
- static final String ACTION_DATA_OUTPUT_PROPS = "output.properties";
- static final String ACTION_DATA_STATS = "stats.properties";
- static final String ACTION_DATA_NEW_ID = "newId";
- static final String ACTION_DATA_ERROR_PROPS = "error.properties";
- public static final String HADOOP2_WORKAROUND_DISTRIBUTED_CACHE = "oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache";
- public static final String PROPAGATION_CONF_XML = "propagation-conf.xml";
- public static final String OOZIE_LAUNCHER_JOB_ID = "oozie.launcher.job.id";
- public static final String ROOT_LOGGER_LEVEL = "rootlogger.log.level";
-
- private void setRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws LauncherException {
- try {
- String jobId = launcherConf.get("mapred.job.id");
- Path path = new Path(actionDir, recoveryId);
- FileSystem fs = FileSystem.get(path.toUri(), launcherConf);
- if (!fs.exists(path)) {
- try {
- java.io.Writer writer = new OutputStreamWriter(fs.create(path));
- writer.write(jobId);
- writer.close();
- }
- catch (IOException ex) {
- failLauncher(0, "IO error", ex);
- }
- }
- else {
- InputStream is = fs.open(path);
- BufferedReader reader = new BufferedReader(new InputStreamReader(is));
- String id = reader.readLine();
- reader.close();
- if (!jobId.equals(id)) {
- failLauncher(0, MessageFormat.format(
- "Hadoop job Id mismatch, action file [{0}] declares Id [{1}] current Id [{2}]", path, id,
- jobId), null);
- }
-
- }
- }
- catch (IOException ex) {
- failLauncher(0, "IO error", ex);
- }
- }
-
- private Configuration appConf;
- private Path actionDir;
- private ScheduledThreadPoolExecutor timer;
-
- private boolean configFailure = false;
- private LauncherException configureFailureEx;
- private Map<String,String> actionData;
-
- public LauncherMapper() {
- actionData = new HashMap<String,String>();
- }
-
- public void configure(Configuration appConf) {
- System.out.println();
- System.out.println("Oozie Launcher starts");
- System.out.println();
- this.appConf = appConf;
- actionDir = new Path(getAppConf().get(OOZIE_ACTION_DIR_PATH));
- String recoveryId = appConf.get(OOZIE_ACTION_RECOVERY_ID, null);
- try {
- setRecoveryId(appConf, actionDir, recoveryId);
- }
- catch (LauncherException ex) {
- System.out.println("Launcher config error "+ex.getMessage());
- configureFailureEx = ex;
- configFailure = true;
- }
- }
-
-
- public void map(K1 key, V1 value, OutputCollector<K2, V2> collector, Reporter reporter) throws IOException {
- SecurityManager initialSecurityManager = System.getSecurityManager();
- try {
- if (configFailure) {
- throw configureFailureEx;
- }
- else {
- String mainClass = getAppConf().get(CONF_OOZIE_ACTION_MAIN_CLASS);
- if (getAppConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
- System.err.println("WARNING, workaround for Hadoop 2.0.2-alpha distributed cached issue (MAPREDUCE-4820) enabled");
- }
- String msgPrefix = "Main class [" + mainClass + "], ";
- int errorCode = 0;
- Throwable errorCause = null;
- String errorMessage = null;
- try {
- new LauncherSecurityManager();
- }
- catch (SecurityException ex) {
- errorMessage = "Could not set LauncherSecurityManager";
- errorCause = ex;
- }
-
- try {
- setupHeartBeater(reporter);
-
- setupMainConfiguration();
-
- // Propagating the conf to use by child job.
- propagateToHadoopConf();
-
- try {
- System.out.println("Starting the execution of prepare actions");
- executePrepare();
- System.out.println("Completed the execution of prepare actions successfully");
- } catch (Exception ex) {
- System.out.println("Prepare execution in the Launcher Mapper has failed");
- throw new LauncherException(ex.getMessage(), ex);
- }
-
- String[] args = getMainArguments(getAppConf());
-
- printContentsOfCurrentDir();
-
- System.out.println();
- System.out.println("Oozie Java/Map-Reduce/Pig action launcher-job configuration");
- System.out.println("=================================================================");
- System.out.println("Workflow job id : " + System.getProperty("oozie.job.id"));
- System.out.println("Workflow action id: " + System.getProperty("oozie.action.id"));
- System.out.println();
- System.out.println("Classpath :");
- System.out.println("------------------------");
- StringTokenizer st = new StringTokenizer(System.getProperty("java.class.path"), ":");
- while (st.hasMoreTokens()) {
- System.out.println(" " + st.nextToken());
- }
- System.out.println("------------------------");
- System.out.println();
- System.out.println("Main class : " + mainClass);
- System.out.println();
- System.out.println("Maximum output : "
- + getAppConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024));
- System.out.println();
- printArgs("Arguments :", args);
-
- System.out.println();
- System.out.println("Java System Properties:");
- System.out.println("------------------------");
- System.getProperties().store(System.out, "");
- System.out.flush();
- System.out.println("------------------------");
- System.out.println();
-
- System.out.println("=================================================================");
- System.out.println();
- System.out.println(">>> Invoking Main class now >>>");
- System.out.println();
- System.out.flush();
-
- try {
- Class klass = getAppConf().getClass(CONF_OOZIE_ACTION_MAIN_CLASS, Object.class);
- Method mainMethod = klass.getMethod("main", String[].class);
- mainMethod.invoke(null, (Object) args);
- }
- catch (InvocationTargetException ex) {
- // Get what actually caused the exception
- Throwable cause = ex.getCause();
- // If we got a JavaMainException from JavaMain, then we need to unwrap it
- if (JavaMain.JavaMainException.class.isInstance(cause)) {
- cause = cause.getCause();
- }
- if (LauncherMainException.class.isInstance(cause)) {
- errorMessage = msgPrefix + "exit code [" +((LauncherMainException)ex.getCause()).getErrorCode()
- + "]";
- errorCause = null;
- }
- else if (SecurityException.class.isInstance(cause)) {
- if (LauncherSecurityManager.getExitInvoked()) {
- System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode()
- + ")");
- System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode()
- + ")");
- // if 0 main() method finished successfully
- // ignoring
- errorCode = LauncherSecurityManager.getExitCode();
- if (errorCode != 0) {
- errorMessage = msgPrefix + "exit code [" + errorCode + "]";
- errorCause = null;
- }
- }
- }
- else {
- throw ex;
- }
- }
- finally {
- System.out.println();
- System.out.println("<<< Invocation of Main class completed <<<");
- System.out.println();
- }
- if (errorMessage == null) {
- handleActionData();
- if (actionData.get(ACTION_DATA_OUTPUT_PROPS) != null) {
- System.out.println();
- System.out.println("Oozie Launcher, capturing output data:");
- System.out.println("=======================");
- System.out.println(actionData.get(ACTION_DATA_OUTPUT_PROPS));
- System.out.println();
- System.out.println("=======================");
- System.out.println();
- }
- if (actionData.get(ACTION_DATA_NEW_ID) != null) {
- System.out.println();
- System.out.println("Oozie Launcher, propagating new Hadoop job id to Oozie");
- System.out.println("=======================");
- System.out.println(actionData.get(ACTION_DATA_NEW_ID));
- System.out.println("=======================");
- System.out.println();
- }
- }
- }
- catch (NoSuchMethodException ex) {
- errorMessage = msgPrefix + "main() method not found";
- errorCause = ex;
- }
- catch (InvocationTargetException ex) {
- errorMessage = msgPrefix + "main() threw exception";
- errorCause = ex.getTargetException();
- }
- catch (Throwable ex) {
- errorMessage = msgPrefix + "exception invoking main()";
- errorCause = ex;
- }
- finally {
- destroyHeartBeater();
- if (errorMessage != null) {
- failLauncher(errorCode, errorMessage, errorCause);
- }
- }
- }
- }
- catch (LauncherException ex) {
- reporter.incrCounter(COUNTER_GROUP, COUNTER_LAUNCHER_ERROR, 1);
- System.out.println();
- System.out.println("Oozie Launcher failed, finishing Hadoop job gracefully");
- System.out.println();
- }
- finally {
- uploadActionDataToHDFS();
- resetSecurityManager(initialSecurityManager);
- }
- }
-
- public void close() throws IOException {
- System.out.println();
- System.out.println("Oozie Launcher ends");
- System.out.println();
- }
-
- /**
- * Pushing all important conf to hadoop conf for the action
- */
- private void propagateToHadoopConf() throws IOException {
- Configuration propagationConf = new Configuration(false);
- if (System.getProperty(OOZIE_ACTION_ID) != null) {
- propagationConf.set(OOZIE_ACTION_ID, System.getProperty(OOZIE_ACTION_ID));
- }
- if (System.getProperty(OOZIE_JOB_ID) != null) {
- propagationConf.set(OOZIE_JOB_ID, System.getProperty(OOZIE_JOB_ID));
- }
- if(System.getProperty(OOZIE_LAUNCHER_JOB_ID) != null) {
- propagationConf.set(OOZIE_LAUNCHER_JOB_ID, System.getProperty(OOZIE_LAUNCHER_JOB_ID));
- }
-
- // loading action conf prepared by Oozie
- Configuration actionConf = LauncherMain.loadActionConf();
-
- if(actionConf.get(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS) != null) {
- propagationConf.set(LauncherMain.MAPREDUCE_JOB_TAGS,
- actionConf.get(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS));
- }
-
- propagationConf.writeXml(new FileWriter(PROPAGATION_CONF_XML));
- Configuration.dumpConfiguration(propagationConf, new OutputStreamWriter(System.out));
- Configuration.addDefaultResource(PROPAGATION_CONF_XML);
- }
-
- protected Configuration getAppConf() {
- return appConf;
- }
-
- private void handleActionData() throws IOException, LauncherException {
- // external child IDs
- String externalChildIdsProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS);
- if (externalChildIdsProp != null) {
- File externalChildIDs = new File(externalChildIdsProp);
- if (externalChildIDs.exists()) {
- actionData.put(ACTION_DATA_EXTERNAL_CHILD_IDS, getLocalFileContentStr(externalChildIDs, "", -1));
- }
- }
-
- // external stats
- String statsProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_STATS);
- if (statsProp != null) {
- File actionStatsData = new File(statsProp);
- if (actionStatsData.exists()) {
- int statsMaxOutputData = getAppConf().getInt(CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, Integer.MAX_VALUE);
- actionData.put(ACTION_DATA_STATS, getLocalFileContentStr(actionStatsData, "Stats", statsMaxOutputData));
- }
- }
-
- // output data
- String outputProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS);
- if (outputProp != null) {
- File actionOutputData = new File(outputProp);
- if (actionOutputData.exists()) {
- int maxOutputData = getAppConf().getInt(CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 2 * 1024);
- actionData.put(ACTION_DATA_OUTPUT_PROPS,
- getLocalFileContentStr(actionOutputData, "Output", maxOutputData));
- }
- }
-
- // id swap
- String newIdProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID);
- if (newIdProp != null) {
- File newId = new File(newIdProp);
- if (newId.exists()) {
- actionData.put(ACTION_DATA_NEW_ID, getLocalFileContentStr(newId, "", -1));
- }
- }
- }
-
- public static String getLocalFileContentStr(File file, String type, int maxLen) throws LauncherException, IOException {
- StringBuffer sb = new StringBuffer();
- FileReader reader = new FileReader(file);
- char[] buffer = new char[2048];
- int read;
- int count = 0;
- while ((read = reader.read(buffer)) > -1) {
- count += read;
- if (maxLen > -1 && count > maxLen) {
- throw new LauncherException(type + " data exceeds its limit ["+ maxLen + "]");
- }
- sb.append(buffer, 0, read);
- }
- reader.close();
- return sb.toString();
- }
-
- private void uploadActionDataToHDFS() throws IOException {
- if (!actionData.isEmpty()) {
- Path finalPath = new Path(actionDir, ACTION_DATA_SEQUENCE_FILE);
- FileSystem fs = FileSystem.get(finalPath.toUri(), getAppConf());
- // upload into sequence file
- System.out.println("Oozie Launcher, uploading action data to HDFS sequence file: "
- + new Path(actionDir, ACTION_DATA_SEQUENCE_FILE).toUri());
-
- SequenceFile.Writer wr = null;
- try {
- wr = SequenceFile.createWriter(fs, getAppConf(), finalPath, Text.class, Text.class);
- if (wr != null) {
- for (Entry<String, String> entry : actionData.entrySet()) {
- wr.append(new Text(entry.getKey()), new Text(entry.getValue()));
- }
- }
- else {
- throw new IOException("SequenceFile.Writer is null for " + finalPath);
- }
- }
- catch(IOException e) {
- e.printStackTrace();
- throw e;
- }
- finally {
- if (wr != null) {
- wr.close();
- }
- }
- }
- }
-
- private void setupMainConfiguration() throws IOException {
- Path pathNew = new Path(new Path(actionDir, ACTION_CONF_XML),
- new Path(new File(ACTION_CONF_XML).getAbsolutePath()));
- FileSystem fs = FileSystem.get(pathNew.toUri(), getAppConf());
- fs.copyToLocalFile(new Path(actionDir, ACTION_CONF_XML), new Path(new File(ACTION_CONF_XML).getAbsolutePath()));
-
- System.setProperty("oozie.launcher.job.id", getAppConf().get("mapred.job.id"));
- System.setProperty(OOZIE_JOB_ID, getAppConf().get(OOZIE_JOB_ID));
- System.setProperty(OOZIE_ACTION_ID, getAppConf().get(OOZIE_ACTION_ID));
- System.setProperty("oozie.action.conf.xml", new File(ACTION_CONF_XML).getAbsolutePath());
- System.setProperty(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS,
- new File(ACTION_DATA_EXTERNAL_CHILD_IDS).getAbsolutePath());
- System.setProperty(ACTION_PREFIX + ACTION_DATA_STATS, new File(ACTION_DATA_STATS).getAbsolutePath());
- System.setProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID, new File(ACTION_DATA_NEW_ID).getAbsolutePath());
- System.setProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, new File(ACTION_DATA_OUTPUT_PROPS).getAbsolutePath());
- System.setProperty(ACTION_PREFIX + ACTION_DATA_ERROR_PROPS, new File(ACTION_DATA_ERROR_PROPS).getAbsolutePath());
- if (getAppConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME) != null) {
- System.setProperty(LauncherMain.OOZIE_JOB_LAUNCH_TIME,
- getAppConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME));
- }
-
- String actionConfigClass = getAppConf().get(OOZIE_ACTION_CONFIG_CLASS);
- if (actionConfigClass != null) {
- System.setProperty(OOZIE_ACTION_CONFIG_CLASS, actionConfigClass);
- }
- }
-
- // Method to execute the prepare actions
- private void executePrepare() throws IOException, LauncherException, ParserConfigurationException, SAXException {
- String prepareXML = getAppConf().get(ACTION_PREPARE_XML);
- if (prepareXML != null) {
- if (!prepareXML.equals("")) {
- Configuration actionConf = new Configuration(getAppConf());
- String actionXml = System.getProperty("oozie.action.conf.xml");
- actionConf.addResource(new Path("file:///", actionXml));
- PrepareActionsDriver.doOperations(prepareXML, actionConf);
- } else {
- System.out.println("There are no prepare actions to execute.");
- }
- }
- }
-
- public static String[] getMainArguments(Configuration conf) {
- String[] args = new String[conf.getInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, 0)];
-
- String[] retArray;
-
- if (conf.getBoolean(CONF_OOZIE_NULL_ARGS_ALLOWED, true)) {
- for (int i = 0; i < args.length; i++) {
- args[i] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
- }
-
- retArray = args;
- } else {
- int pos = 0;
- for (int i = 0; i < args.length; i++) {
- String arg = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
- if (!Strings.isNullOrEmpty(arg)) {
- args[pos++] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
- }
- }
-
- // this is to skip null args, that is <arg></arg> in the workflow XML -- in this case,
- // args[] might look like {"arg1", "arg2", null, null} at this point
- retArray = new String[pos];
- System.arraycopy(args, 0, retArray, 0, pos);
- }
-
- return retArray;
- }
-
- private void setupHeartBeater(Reporter reporter) {
- timer = new ScheduledThreadPoolExecutor(1);
- timer.scheduleAtFixedRate(new LauncherMapper(reporter), 0, 30, TimeUnit.SECONDS);
- }
-
- private void destroyHeartBeater() {
- timer.shutdownNow();
- }
-
- private Reporter reporter;
-
- private LauncherMapper(Reporter reporter) {
- this.reporter = reporter;
- }
-
- @Override
- public void run() {
- System.out.println("Heart beat");
- reporter.progress();
- }
-
- private void failLauncher(int errorCode, String reason, Throwable ex) throws LauncherException {
- if (ex != null) {
- reason += ", " + ex.getMessage();
- }
- Properties errorProps = new Properties();
- errorProps.setProperty("error.code", Integer.toString(errorCode));
- errorProps.setProperty("error.reason", reason);
- if (ex != null) {
- if (ex.getMessage() != null) {
- errorProps.setProperty("exception.message", ex.getMessage());
- }
- StringWriter sw = new StringWriter();
- PrintWriter pw = new PrintWriter(sw);
- ex.printStackTrace(pw);
- pw.close();
- errorProps.setProperty("exception.stacktrace", sw.toString());
- }
- StringWriter sw = new StringWriter();
- try {
- errorProps.store(sw, "");
- sw.close();
- actionData.put(ACTION_DATA_ERROR_PROPS, sw.toString());
-
- // external child IDs
- String externalChildIdsProp = System.getProperty(ACTION_PREFIX + ACTION_DATA_EXTERNAL_CHILD_IDS);
- if (externalChildIdsProp != null) {
- File externalChildIDs = new File(externalChildIdsProp);
- if (externalChildIDs.exists()) {
- actionData.put(ACTION_DATA_EXTERNAL_CHILD_IDS, getLocalFileContentStr(externalChildIDs, "", -1));
- }
- }
- }
- catch (IOException ioe) {
- throw new LauncherException(ioe.getMessage(), ioe);
- }
- finally {
- System.out.print("Failing Oozie Launcher, " + reason + "\n");
- System.err.print("Failing Oozie Launcher, " + reason + "\n");
- if (ex != null) {
- ex.printStackTrace(System.out);
- ex.printStackTrace(System.err);
- }
- }
- throw new LauncherException(reason, ex);
- }
-
- /**
- * Print files and directories in current directory. Will list files in the sub-directory (only 1 level deep)
- */
- protected void printContentsOfCurrentDir() {
- File folder = new File(".");
- System.out.println();
- System.out.println("Files in current dir:" + folder.getAbsolutePath());
- System.out.println("======================");
-
- File[] listOfFiles = folder.listFiles();
-
- if (listOfFiles != null) {
- for (File fileName : listOfFiles) {
- if (fileName.isFile()) {
- System.out.println("File: " + fileName.getName());
- }
- else if (fileName.isDirectory()) {
- System.out.println("Dir: " + fileName.getName());
- File subDir = new File(fileName.getName());
- File[] moreFiles = subDir.listFiles();
-
- if (moreFiles != null) {
- for (File subFileName : moreFiles) {
- if (subFileName.isFile()) {
- System.out.println(" File: " + subFileName.getName());
- }
- else if (subFileName.isDirectory()) {
- System.out.println(" Dir: " + subFileName.getName());
- }
- }
- }
- }
- }
- }
- }
-
- private void resetSecurityManager(SecurityManager initialSecurityManager) {
- try {
- SecurityManager prev = System.getSecurityManager();
- System.setSecurityManager(initialSecurityManager);
- System.out
- .println("Successfully reset security manager from " + prev + " to " + System.getSecurityManager());
- }
- catch (Throwable t) {
- System.err.println("Failed to reset security manager: " + t.getMessage());
- t.printStackTrace(System.err);
- }
- }
-
- /**
- * Print arguments to standard output stream. Mask out argument values to option with name 'password' in them.
- * @param banner source banner
- * @param args arguments to be printed
- */
- public static void printArgs(String banner, String[] args) {
- System.out.println(banner);
- boolean maskNextArg = false;
- for (String arg : args) {
- if (arg == null) {
- arg = "null"; // prevent NPE in pwd masking
- }
-
- if (maskNextArg) {
- System.out.println(" " + "********");
- maskNextArg = false;
- }
- else {
- System.out.println(" " + arg);
- if (arg.toLowerCase().contains("password")) {
- maskNextArg = true;
- }
- }
- }
- }
-}
-
-class LauncherSecurityManager extends SecurityManager {
- private static boolean exitInvoked;
- private static int exitCode;
- private SecurityManager securityManager;
-
- public LauncherSecurityManager() {
- reset();
- securityManager = System.getSecurityManager();
- System.setSecurityManager(this);
- }
-
- @Override
- public void checkPermission(Permission perm, Object context) {
- if (securityManager != null) {
- // check everything with the original SecurityManager
- securityManager.checkPermission(perm, context);
- }
- }
-
- @Override
- public void checkPermission(Permission perm) {
- if (securityManager != null) {
- // check everything with the original SecurityManager
- securityManager.checkPermission(perm);
- }
- }
-
- @Override
- public void checkExit(int status) throws SecurityException {
- exitInvoked = true;
- exitCode = status;
- throw new SecurityException("Intercepted System.exit(" + status + ")");
- }
-
- public static boolean getExitInvoked() {
- return exitInvoked;
- }
-
- public static int getExitCode() {
- return exitCode;
- }
-
- public static void reset() {
- exitInvoked = false;
- exitCode = 0;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
index e0974e8..77aaa62 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
@@ -66,7 +66,7 @@ public class MapReduceMain extends LauncherMain {
logMasking("Map-Reduce job configuration:", new ArrayList<String>(), maskedJobConf);
- File idFile = new File(System.getProperty(LauncherMapper.ACTION_PREFIX + LauncherMapper.ACTION_DATA_NEW_ID));
+ File idFile = new File(System.getProperty(LauncherAMUtils.ACTION_PREFIX + LauncherAMUtils.ACTION_DATA_NEW_ID));
System.out.println("Submitting Oozie action Map-Reduce job");
System.out.println();
// submitting job
@@ -143,7 +143,7 @@ public class MapReduceMain extends LauncherMain {
* @throws OozieActionConfiguratorException
*/
private static void runConfigClass(JobConf actionConf) throws OozieActionConfiguratorException {
- String configClass = actionConf.get(LauncherMapper.OOZIE_ACTION_CONFIG_CLASS);
+ String configClass = actionConf.get(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS);
if (configClass != null) {
try {
Class<?> klass = Class.forName(configClass);
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java
deleted file mode 100644
index 3895538..0000000
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.oozie.action.hadoop;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.ObjectWritable;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordReader;
-import org.apache.hadoop.mapred.Reporter;
-/**
- * Dummy input format implementation of Oozie launcher jobs. It returns only one record.
- */
-public class OozieLauncherInputFormat implements InputFormat<Object, Object> {
-
- boolean isReadingDone = false;
-
- public RecordReader<Object, Object> getRecordReader(InputSplit arg0, JobConf arg1, Reporter arg2)
- throws IOException {
- return new RecordReader<Object, Object>() {
-
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public float getProgress() throws IOException {
- if (isReadingDone) {
- return 1.0f;
- }
- else
- return 0.0f;
- }
-
- @Override
- public Object createKey() {
- return new ObjectWritable();
- }
-
- @Override
- public Object createValue() {
- return new ObjectWritable();
- }
-
- @Override
- public long getPos() throws IOException {
- if (isReadingDone) {
- return 1;
- }
- else {
- return 0;
- }
- }
-
- @Override
- public boolean next(Object arg0, Object arg1) throws IOException {
- if (isReadingDone) {
- return false;
- }
- else {
- isReadingDone = true;
- return true;
- }
- }
-
- };
- }
-
- @Override
- public InputSplit[] getSplits(JobConf arg0, int arg1) throws IOException {
- return new InputSplit[] { new EmptySplit() };
- }
-
- /**
- * Empty Split implementation.
- */
- public static class EmptySplit implements InputSplit {
-
- @Override
- public void write(DataOutput out) throws IOException {
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- }
-
- @Override
- public long getLength() {
- return 0L;
- }
-
- @Override
- public String[] getLocations() {
- return new String[0];
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java
index f13e8a0..27da385 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputCommitter.java
@@ -29,13 +29,13 @@ import org.apache.hadoop.mapred.TaskAttemptContext;
public class OozieLauncherOutputCommitter extends OutputCommitter {
public OozieLauncherOutputCommitter() {
- File propConf = new File(LauncherMapper.PROPAGATION_CONF_XML);
+ File propConf = new File(LauncherAMUtils.PROPAGATION_CONF_XML);
if (!propConf.exists()) {
try {
propConf.createNewFile();
}
catch (IOException e) {
- System.out.println("Failed to create " + LauncherMapper.PROPAGATION_CONF_XML);
+ System.out.println("Failed to create " + LauncherAMUtils.PROPAGATION_CONF_XML);
e.printStackTrace(System.err);
}
}
@@ -76,4 +76,4 @@ public class OozieLauncherOutputCommitter extends OutputCommitter {
public boolean isRecoverySupported(JobContext jobContext) throws IOException {
return isRecoverySupported();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java
deleted file mode 100644
index 9e18dd9..0000000
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherOutputFormat.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import java.io.IOException;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hadoop.mapred.RecordWriter;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.util.Progressable;
-
-public class OozieLauncherOutputFormat implements OutputFormat<Object, Object>{
-
- @Override
- public RecordWriter<Object, Object> getRecordWriter(FileSystem ignored, JobConf job, String name,
- Progressable progress) throws IOException {
- return new RecordWriter<Object, Object>() {
- @Override
- public void write(Object key, Object value) throws IOException {
- }
- @Override
- public void close(Reporter reporter) throws IOException {
- }
- };
- }
-
- @Override
- public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
index 0ee35e8..bde7f1d 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/ShellMain.java
@@ -52,7 +52,7 @@ public class ShellMain extends LauncherMain {
private static String LOG4J_PROPERTIES = "log4j.properties";
/**
- * @param args Invoked from LauncherMapper:map()
+ * @param args Invoked from LauncherAMUtils:map()
* @throws Exception
*/
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
index 9cdedb7..37af3dd 100644
--- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
+++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAM.java
@@ -137,7 +137,7 @@ public class TestLauncherAM {
@Before
public void setup() throws Exception {
configureMocksForHappyPath();
- launcherJobConfig.set(LauncherMapper.OOZIE_ACTION_RECOVERY_ID, "1");
+ launcherJobConfig.set(LauncherAMUtils.OOZIE_ACTION_RECOVERY_ID, "1");
instantiateLauncher();
}
@@ -399,7 +399,7 @@ public class TestLauncherAM {
@Test
public void testRecoveryIdNotSet() throws Exception {
- launcherJobConfig.unset(LauncherMapper.OOZIE_ACTION_RECOVERY_ID);
+ launcherJobConfig.unset(LauncherAMUtils.OOZIE_ACTION_RECOVERY_ID);
instantiateLauncher();
executeLauncher();
@@ -638,4 +638,4 @@ public class TestLauncherAM {
return this;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMUtils.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMUtils.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMUtils.java
new file mode 100644
index 0000000..43c4914
--- /dev/null
+++ b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherAMUtils.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import static org.apache.oozie.action.hadoop.LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_COUNT;
+import static org.apache.oozie.action.hadoop.LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.anyBoolean;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RunningJob;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.google.common.collect.Lists;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestLauncherAMUtils {
+ @Mock
+ private Configuration conf; // we have to use mock, because conf.set(null) throws exception
+
+ @Test
+ public void testArgsHandlingWithoutNullsAndNullsNotAllowed() {
+ setupConf(Lists.newArrayList("a", "b", "c"));
+ setEnableNullArgsAllowed(false);
+
+ String args[] = LauncherAMUtils.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args));
+ }
+
+ @Test
+ public void testHandlingWhenArgsContainNullsAndNullsNotAllowed() {
+ setupConf(Lists.newArrayList("a", null, "b", null, "c"));
+ setEnableNullArgsAllowed(false);
+
+ String args[] = LauncherAMUtils.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args));
+ }
+
+ @Test
+ public void testArgsHandlingWhenArgsContainsNullsOnlyAndNullsNotAllowed() {
+ setupConf(Lists.<String>newArrayList(null, null, null));
+ setEnableNullArgsAllowed(false);
+
+ String args[] = LauncherAMUtils.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] {}, args));
+ }
+
+ @Test
+ public void testArgsHandlingWhenArgsContainsOneNullAndNullsNotAllowed() {
+ setupConf(Lists.<String>newArrayList((String) null));
+ setEnableNullArgsAllowed(false);
+
+ String args[] = LauncherAMUtils.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] {}, args));
+ }
+
+ @Test
+ public void testHandlingWhenArgsContainNullsAndNullAllowed() {
+ setupConf(Lists.newArrayList("a", null, "b", null, "c"));
+ setEnableNullArgsAllowed(true);
+
+ String args[] = LauncherAMUtils.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] { "a", null, "b", null, "c"}, args));
+ }
+
+ @Test
+ public void testArgsHandlingWhenArgsContainsOneNullAndNullsAllowed() {
+ setupConf(Lists.<String>newArrayList((String) null));
+ setEnableNullArgsAllowed(true);
+
+ String args[] = LauncherAMUtils.getMainArguments(conf);
+
+ assertTrue(Arrays.equals(new String[] { null }, args));
+ }
+
+ private void setupConf(List<String> argList) {
+ int argCount = argList.size();
+
+ given(conf.getInt(eq(CONF_OOZIE_ACTION_MAIN_ARG_COUNT), eq(0))).willReturn(argCount);
+
+ for (int i = 0; i < argCount; i++) {
+ given(conf.get(eq(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i))).willReturn(argList.get(i));
+ }
+ }
+
+ private void setEnableNullArgsAllowed(boolean nullArgsAllowed) {
+ given(conf.getBoolean(eq(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED), anyBoolean())).willReturn(nullArgsAllowed);
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java b/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
deleted file mode 100644
index 51b1d6f..0000000
--- a/sharelib/oozie/src/test/java/org/apache/oozie/action/hadoop/TestLauncherMapper.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_COUNT;
-import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.BDDMockito.given;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Matchers.anyBoolean;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mock;
-import org.mockito.runners.MockitoJUnitRunner;
-
-import com.google.common.collect.Lists;
-
-@RunWith(MockitoJUnitRunner.class)
-public class TestLauncherMapper {
- @Mock
- private Configuration conf; // we have to use mock, because conf.set(null) throws exception
-
- @Test
- public void testArgsHandlingWithoutNullsAndNullsNotAllowed() {
- setupConf(Lists.newArrayList("a", "b", "c"));
- setEnableNullArgsAllowed(false);
-
- String args[] = LauncherMapper.getMainArguments(conf);
-
- assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args));
- }
-
- @Test
- public void testHandlingWhenArgsContainNullsAndNullsNotAllowed() {
- setupConf(Lists.newArrayList("a", null, "b", null, "c"));
- setEnableNullArgsAllowed(false);
-
- String args[] = LauncherMapper.getMainArguments(conf);
-
- assertTrue(Arrays.equals(new String[] { "a", "b", "c"}, args));
- }
-
- @Test
- public void testArgsHandlingWhenArgsContainsNullsOnlyAndNullsNotAllowed() {
- setupConf(Lists.<String>newArrayList(null, null, null));
- setEnableNullArgsAllowed(false);
-
- String args[] = LauncherMapper.getMainArguments(conf);
-
- assertTrue(Arrays.equals(new String[] {}, args));
- }
-
- @Test
- public void testArgsHandlingWhenArgsContainsOneNullAndNullsNotAllowed() {
- setupConf(Lists.<String>newArrayList((String) null));
- setEnableNullArgsAllowed(false);
-
- String args[] = LauncherMapper.getMainArguments(conf);
-
- assertTrue(Arrays.equals(new String[] {}, args));
- }
-
- @Test
- public void testHandlingWhenArgsContainNullsAndNullAllowed() {
- setupConf(Lists.newArrayList("a", null, "b", null, "c"));
- setEnableNullArgsAllowed(true);
-
- String args[] = LauncherMapper.getMainArguments(conf);
-
- assertTrue(Arrays.equals(new String[] { "a", null, "b", null, "c"}, args));
- }
-
- @Test
- public void testArgsHandlingWhenArgsContainsOneNullAndNullsAllowed() {
- setupConf(Lists.<String>newArrayList((String) null));
- setEnableNullArgsAllowed(true);
-
- String args[] = LauncherMapper.getMainArguments(conf);
-
- assertTrue(Arrays.equals(new String[] { null }, args));
- }
-
- private void setupConf(List<String> argList) {
- int argCount = argList.size();
-
- given(conf.getInt(eq(CONF_OOZIE_ACTION_MAIN_ARG_COUNT), eq(0))).willReturn(argCount);
-
- for (int i = 0; i < argCount; i++) {
- given(conf.get(eq(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i))).willReturn(argList.get(i));
- }
- }
-
- private void setEnableNullArgsAllowed(boolean nullArgsAllowed) {
- given(conf.getBoolean(eq(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED), anyBoolean())).willReturn(nullArgsAllowed);
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index b6b30eb..4882981 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -53,7 +53,7 @@ import java.util.regex.Pattern;
public class PigMain extends LauncherMain {
private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<>();
public static final int STRING_BUFFER_SIZE = 100;
- public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherMapper.ACTION_PREFIX + "pig.log.expandedscript";
+ public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherAMUtils.ACTION_PREFIX + "pig.log.expandedscript";
@VisibleForTesting
static final Pattern[] PIG_JOB_IDS_PATTERNS = {
@@ -162,7 +162,7 @@ public class PigMain extends LauncherMain {
String logFile = new File("pig-oozie-" + hadoopJobId + ".log").getAbsolutePath();
String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO");
- String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ String rootLogLevel = actionConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO");
// append required PIG properties to the default hadoop log4j file
log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A, B");
@@ -378,29 +378,12 @@ public class PigMain extends LauncherMain {
}
}
else {
- try {
System.out.println("Run pig script using Main.main() for Pig version before 0.8");
Main.main(args);
- }
- catch (SecurityException ex) {
- if (resetSecurityManager) {
- LauncherSecurityManager.reset();
- }
- else {
- if (LauncherSecurityManager.getExitInvoked()) {
- if (LauncherSecurityManager.getExitCode() != 0) {
- if (pigLog != null) {
- handleError(pigLog);
- }
- throw ex;
- }
- }
- }
- }
}
}
- // write external data(stats, hadoopIds) to the file which will be read by the LauncherMapper
+ // write external data(stats, hadoopIds) to the file which will be read by the LauncherAMUtils
private static void writeExternalData(String data, File f) throws IOException {
BufferedWriter out = null;
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
index df31655..7884cdd 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
@@ -153,7 +153,7 @@ public class PigMainWithOldAPI extends LauncherMain {
String logFile = new File("pig-oozie-" + hadoopJobId + ".log").getAbsolutePath();
String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO");
- String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ String rootLogLevel = actionConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO");
// append required PIG properties to the default hadoop log4j file
log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A, B");
@@ -200,22 +200,24 @@ public class PigMainWithOldAPI extends LauncherMain {
System.out.println(">>> Invoking Pig command line now >>>");
System.out.println();
System.out.flush();
-
+ LauncherAM.LauncherSecurityManager launcherSecurityManager = new LauncherAM.LauncherSecurityManager();
+ launcherSecurityManager.enable();
try {
System.out.println();
runPigJob(new String[] { "-version" });
}
catch (SecurityException ex) {
- LauncherSecurityManager.reset();
+ launcherSecurityManager.reset();
}
System.out.println();
System.out.flush();
+
try {
runPigJob(arguments.toArray(new String[arguments.size()]));
}
catch (SecurityException ex) {
- if (LauncherSecurityManager.getExitInvoked()) {
- if (LauncherSecurityManager.getExitCode() != 0) {
+ if (launcherSecurityManager.getExitInvoked()) {
+ if (launcherSecurityManager.getExitCode() != 0) {
System.err.println();
System.err.println("Pig logfile dump:");
System.err.println();
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
index 7c3c5bb..75727d2 100644
--- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
+++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java
@@ -225,7 +225,7 @@ public class TestPigActionExecutor extends ActionExecutorTestCase {
Context context = createContext(actionXml);
submitAction(context);
FileSystem fs = getFileSystem();
- FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML));
+ FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherAMUtils.ACTION_CONF_XML));
XConfiguration conf = new XConfiguration();
conf.addResource(os);
assertNull(conf.get("oozie.HadoopAccessorService.created"));
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
index 22e7952..9a185c9 100644
--- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
+++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java
@@ -115,7 +115,6 @@ public class TestPigMain extends PigTestCase {
assertEquals(props.getProperty("oozie.pig.args.size"), "1");
File pigProps = new File(classPathDir, "pig.properties");
- new LauncherSecurityManager();
String user = System.getProperty("user.name");
try {
Writer wr = new FileWriter(pigProps);
@@ -123,20 +122,6 @@ public class TestPigMain extends PigTestCase {
wr.close();
PigMain.main(null);
}
- catch (SecurityException ex) {
- if (LauncherSecurityManager.getExitInvoked()) {
- System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")");
- System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")");
- if (failOnException) {
- if (LauncherSecurityManager.getExitCode() != 0) {
- fail();
- }
- }
- }
- else {
- throw ex;
- }
- }
finally {
pigProps.delete();
System.setProperty("user.name", user);
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java
index 74de433..d5a5797 100644
--- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java
+++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMainWithOldAPI.java
@@ -36,9 +36,12 @@ import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.Writer;
import java.net.URL;
+import java.security.Permission;
import java.util.Properties;
import java.util.concurrent.Callable;
+import static groovy.util.GroovyTestCase.assertEquals;
+
/**
* Test PigMainWithOldAPI class should run a Pig script and write results to
@@ -118,29 +121,18 @@ public class TestPigMainWithOldAPI extends XFsTestCase implements Callable<Void>
assertEquals(props.getProperty("oozie.pig.args.size"), "1");
File pigProps = new File(classPathDir, "pig.properties");
- new LauncherSecurityManager();
String user = System.getProperty("user.name");
ByteArrayOutputStream data = new ByteArrayOutputStream();
PrintStream oldPrintStream = System.out;
System.setOut(new PrintStream(data));
+
+
try {
Writer wr = new FileWriter(pigProps);
props.store(wr, "");
wr.close();
PigMainWithOldAPI.main(null);
}
- catch (SecurityException ex) {
- if (LauncherSecurityManager.getExitInvoked()) {
- System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")");
- System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")");
- if (LauncherSecurityManager.getExitCode() != 0) {
- fail("Exit code should be 0");
- }
- }
- else {
- throw ex;
- }
- }
finally {
pigProps.delete();
System.setProperty("user.name", user);
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
----------------------------------------------------------------------
diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
index 674839a..8aa8343 100644
--- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
+++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java
@@ -191,7 +191,7 @@ public class SparkMain extends LauncherMain {
final String logFile = new File("spark-oozie-" + hadoopJobId + ".log").getAbsolutePath();
final String logLevel = actionConf.get("oozie.spark.log.level", "INFO");
- final String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ final String rootLogLevel = actionConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO");
log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A");
log4jProperties.setProperty("log4j.logger.org.apache.spark", logLevel + ", A, jobid");
@@ -272,4 +272,4 @@ public class SparkMain extends LauncherMain {
// to null.
HiveConf.setHiveSiteLocation(HiveConf.class.getClassLoader().getResource("hive-site.xml"));
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index 416f1ec..e889ef9 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -22,9 +22,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-import java.net.URL;
import java.util.Map;
-import java.util.Properties;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
@@ -116,7 +114,7 @@ public class SqoopMain extends LauncherMain {
String logFile = new File("sqoop-oozie-" + hadoopJobId + ".log").getAbsolutePath();
String logLevel = sqoopConf.get("oozie.sqoop.log.level", "INFO");
- String rootLogLevel = sqoopConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ String rootLogLevel = sqoopConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO");
log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A");
log4jProperties.setProperty("log4j.logger.org.apache.sqoop", logLevel + ", A");
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
index 045f174..3b38042 100644
--- a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
+++ b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
@@ -616,7 +616,7 @@ public class TestMapReduceActionExecutor extends ActionExecutorTestCase {
final Map<String, String> actionData = LauncherHelper.getActionData(fs, context.getActionDir(),
context.getProtoActionConf());
- Properties errorProps = PropertiesUtils.stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS));
+ Properties errorProps = PropertiesUtils.stringToProperties(actionData.get(LauncherAMUtils.ACTION_DATA_ERROR_PROPS));
assertEquals("An Exception occurred while instantiating the action config class",
errorProps.getProperty("exception.message"));
assertTrue(errorProps.getProperty("exception.stacktrace").startsWith(OozieActionConfiguratorException.class.getName()));
@@ -646,7 +646,7 @@ public class TestMapReduceActionExecutor extends ActionExecutorTestCase {
final Map<String, String> actionData = LauncherHelper.getActionData(fs, context.getActionDir(),
context.getProtoActionConf());
- Properties errorProps = PropertiesUtils.stringToProperties(actionData.get(LauncherMapper.ACTION_DATA_ERROR_PROPS));
+ Properties errorProps = PropertiesUtils.stringToProperties(actionData.get(LauncherAMUtils.ACTION_DATA_ERROR_PROPS));
assertEquals("doh", errorProps.getProperty("exception.message"));
assertTrue(errorProps.getProperty("exception.stacktrace").startsWith(OozieActionConfiguratorException.class.getName()));
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
----------------------------------------------------------------------
diff --git a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
index a57580e..ef268b2 100644
--- a/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
+++ b/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestStreamingMain.java
@@ -74,7 +74,7 @@ public class TestStreamingMain extends MainTestCase {
StreamingMain.main(null);
assertTrue(newId.exists());
- assertNotNull(LauncherMapper.getLocalFileContentStr(newId, "", -1));
+ assertNotNull(LauncherAMUtils.getLocalFileContentStr(newId, "", -1));
return null;
}
[2/2] oozie git commit: OOZIE-2918 Delete LauncherMapper and its test
(asasvari via pbacsko)
Posted by pb...@apache.org.
OOZIE-2918 Delete LauncherMapper and its test (asasvari via pbacsko)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/c09561ff
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/c09561ff
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/c09561ff
Branch: refs/heads/master
Commit: c09561ff263f38ff4ecdf81f71359ae2ae4c90d5
Parents: 79ccfdd
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Tue Jul 4 14:30:11 2017 +0200
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Tue Jul 4 14:30:11 2017 +0200
----------------------------------------------------------------------
.../action/hadoop/DistcpActionExecutor.java | 2 +-
.../oozie/action/hadoop/FsActionExecutor.java | 2 +-
.../oozie/action/hadoop/FsELFunctions.java | 2 +-
.../action/hadoop/Hive2ActionExecutor.java | 4 +-
.../oozie/action/hadoop/HiveActionExecutor.java | 2 +-
.../oozie/action/hadoop/JavaActionExecutor.java | 13 +-
.../oozie/action/hadoop/LauncherHelper.java | 62 +-
.../hadoop/LauncherInputFormatClassLocator.java | 84 ---
.../action/hadoop/MapReduceActionExecutor.java | 27 +-
.../oozie/action/hadoop/PigActionExecutor.java | 2 +-
.../action/hadoop/ShellActionExecutor.java | 2 +-
.../action/hadoop/SparkActionExecutor.java | 4 +-
.../action/hadoop/SqoopActionExecutor.java | 3 +-
core/src/main/resources/oozie-default.xml | 2 +-
.../oozie/action/hadoop/LauncherMainTester.java | 2 +-
.../oozie/action/hadoop/TestDistcpMain.java | 6 +-
.../action/hadoop/TestJavaActionExecutor.java | 8 +-
.../oozie/action/hadoop/TestLauncher.java | 245 -------
.../TestLauncherInputFormatClassLocator.java | 57 --
.../oozie/action/hadoop/TestMapReduceMain.java | 2 +-
.../action/hadoop/TestShellActionExecutor.java | 2 +-
.../oozie/service/TestConfigurationService.java | 7 +-
release-log.txt | 1 +
.../apache/oozie/action/hadoop/DistcpMain.java | 4 +-
.../apache/oozie/action/hadoop/HiveMain.java | 2 +-
.../action/hadoop/TestHiveActionExecutor.java | 6 +-
.../oozie/action/hadoop/TestHiveMain.java | 13 +-
.../action/hadoop/FSLauncherURIHandler.java | 2 +-
.../apache/oozie/action/hadoop/JavaMain.java | 2 +-
.../apache/oozie/action/hadoop/LauncherAM.java | 9 +-
.../oozie/action/hadoop/LauncherAMUtils.java | 108 +++
.../oozie/action/hadoop/LauncherMapper.java | 715 -------------------
.../oozie/action/hadoop/MapReduceMain.java | 4 +-
.../action/hadoop/OozieLauncherInputFormat.java | 119 ---
.../hadoop/OozieLauncherOutputCommitter.java | 6 +-
.../hadoop/OozieLauncherOutputFormat.java | 48 --
.../apache/oozie/action/hadoop/ShellMain.java | 2 +-
.../oozie/action/hadoop/TestLauncherAM.java | 6 +-
.../action/hadoop/TestLauncherAMUtils.java | 129 ++++
.../oozie/action/hadoop/TestLauncherMapper.java | 117 ---
.../org/apache/oozie/action/hadoop/PigMain.java | 23 +-
.../oozie/action/hadoop/PigMainWithOldAPI.java | 12 +-
.../action/hadoop/TestPigActionExecutor.java | 2 +-
.../apache/oozie/action/hadoop/TestPigMain.java | 15 -
.../action/hadoop/TestPigMainWithOldAPI.java | 18 +-
.../apache/oozie/action/hadoop/SparkMain.java | 4 +-
.../apache/oozie/action/hadoop/SqoopMain.java | 4 +-
.../hadoop/TestMapReduceActionExecutor.java | 4 +-
.../oozie/action/hadoop/TestStreamingMain.java | 2 +-
49 files changed, 355 insertions(+), 1562 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
index 20f47d5..961994e 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/DistcpActionExecutor.java
@@ -113,6 +113,6 @@ public class DistcpActionExecutor extends JavaActionExecutor{
@Override
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
- return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS);
+ return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, CONF_OOZIE_DISTCP_ACTION_MAIN_CLASS);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
index 0515db6..c2a6192 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsActionExecutor.java
@@ -65,7 +65,7 @@ public class FsActionExecutor extends ActionExecutor {
public FsActionExecutor() {
super(ACTION_TYPE);
- maxGlobCount = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX);
+ maxGlobCount = ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX);
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java
index 295f81a..031be25 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/FsELFunctions.java
@@ -170,7 +170,7 @@ public class FsELFunctions {
int count = 0;
int globMax = Integer.MAX_VALUE;
public FSPathFilter() {
- globMax = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX);
+ globMax = ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX);
}
@Override
public boolean accept(Path p) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
index 35277ae..0480f06 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/Hive2ActionExecutor.java
@@ -27,7 +27,7 @@ import org.jdom.Namespace;
import java.util.ArrayList;
import java.util.List;
-import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
+import static org.apache.oozie.action.hadoop.LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS;
public class Hive2ActionExecutor extends ScriptLanguageActionExecutor {
@@ -142,4 +142,4 @@ public class Hive2ActionExecutor extends ScriptLanguageActionExecutor {
return new String[] { "hive-site.xml" };
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
index 22378fc..d8c613b 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/HiveActionExecutor.java
@@ -30,7 +30,7 @@ import org.jdom.Namespace;
import java.util.ArrayList;
import java.util.List;
-import static org.apache.oozie.action.hadoop.LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS;
+import static org.apache.oozie.action.hadoop.LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS;
public class HiveActionExecutor extends ScriptLanguageActionExecutor {
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index eac8e62..fa83f1e 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -199,7 +199,7 @@ public class JavaActionExecutor extends ActionExecutor {
maxExternalStatsSize = ConfigurationService.getInt(JavaActionExecutor.MAX_EXTERNAL_STATS_SIZE);
maxExternalStatsSize = (maxExternalStatsSize == -1) ? Integer.MAX_VALUE : maxExternalStatsSize;
//Get the limit for the maximum number of globbed files/dirs for FS operation
- maxFSGlobMax = ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX);
+ maxFSGlobMax = ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX);
registerError(UnknownHostException.class.getName(), ActionExecutorException.ErrorType.TRANSIENT, "JA001");
registerError(AccessControlException.class.getName(), ActionExecutorException.ErrorType.NON_TRANSIENT,
@@ -414,8 +414,8 @@ public class JavaActionExecutor extends ActionExecutor {
* @param actionConf
*/
void setRootLoggerLevel(Configuration actionConf) {
- String oozieActionTypeRootLogger = "oozie.action." + getType() + LauncherMapper.ROOT_LOGGER_LEVEL;
- String oozieActionRootLogger = "oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL;
+ String oozieActionTypeRootLogger = "oozie.action." + getType() + LauncherAMUtils.ROOT_LOGGER_LEVEL;
+ String oozieActionRootLogger = "oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL;
// check if root log level has already mentioned in action configuration
String rootLogLevel = actionConf.get(oozieActionTypeRootLogger, actionConf.get(oozieActionRootLogger));
@@ -842,11 +842,8 @@ public class JavaActionExecutor extends ActionExecutor {
}
LauncherHelper.setupMainArguments(launcherJobConf, args);
// backward compatibility flag - see OOZIE-2872
- if (ConfigurationService.getBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED)) {
- launcherJobConf.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, true);
- } else {
- launcherJobConf.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, false);
- }
+ boolean nullArgsAllowed = ConfigurationService.getBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED);
+ launcherJobConf.setBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed);
// Make mapred.child.java.opts and mapreduce.map.java.opts equal, but give values from the latter priority; also append
// <java-opt> and <java-opts> and give those highest priority
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
index f80141c..f9e0aef 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
@@ -55,8 +55,6 @@ public class LauncherHelper {
public static final String OOZIE_ACTION_YARN_TAG = "oozie.action.yarn.tag";
- private static final LauncherInputFormatClassLocator launcherInputFormatClassLocator = new LauncherInputFormatClassLocator();
-
public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId)
throws HadoopAccessorException, IOException {
String jobId = null;
@@ -78,7 +76,7 @@ public class LauncherHelper {
// Only set the javaMainClass if its not null or empty string, this way the user can override the action's main class via
// <configuration> property
if (javaMainClass != null && !javaMainClass.equals("")) {
- launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
+ launcherConf.set(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
}
}
@@ -89,14 +87,14 @@ public class LauncherHelper {
}
public static void setupMainArguments(Configuration launcherConf, String[] args) {
- launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length);
+ launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length);
for (int i = 0; i < args.length; i++) {
- launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]);
+ launcherConf.set(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]);
}
}
public static void setupMaxOutputData(Configuration launcherConf, int maxOutputData) {
- launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData);
+ launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData);
}
/**
@@ -106,7 +104,7 @@ public class LauncherHelper {
* @param maxStatsData the maximum allowed size of stats data
*/
public static void setupMaxExternalStatsSize(Configuration launcherConf, int maxStatsData){
- launcherConf.setInt(LauncherMapper.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, maxStatsData);
+ launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, maxStatsData);
}
/**
@@ -116,20 +114,20 @@ public class LauncherHelper {
* @param fsGlobMax the maximum number of files/dirs for FS operation
*/
public static void setupMaxFSGlob(Configuration launcherConf, int fsGlobMax){
- launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax);
+ launcherConf.setInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax);
}
public static void setupLauncherInfo(Configuration launcherConf, String jobId, String actionId, Path actionDir,
String recoveryId, Configuration actionConf, String prepareXML) throws IOException, HadoopAccessorException {
- launcherConf.set(LauncherMapper.OOZIE_JOB_ID, jobId);
- launcherConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId);
- launcherConf.set(LauncherMapper.OOZIE_ACTION_DIR_PATH, actionDir.toString());
- launcherConf.set(LauncherMapper.OOZIE_ACTION_RECOVERY_ID, recoveryId);
- launcherConf.set(LauncherMapper.ACTION_PREPARE_XML, prepareXML);
+ launcherConf.set(LauncherAMUtils.OOZIE_JOB_ID, jobId);
+ launcherConf.set(LauncherAMUtils.OOZIE_ACTION_ID, actionId);
+ launcherConf.set(LauncherAMUtils.OOZIE_ACTION_DIR_PATH, actionDir.toString());
+ launcherConf.set(LauncherAMUtils.OOZIE_ACTION_RECOVERY_ID, recoveryId);
+ launcherConf.set(LauncherAMUtils.ACTION_PREPARE_XML, prepareXML);
- actionConf.set(LauncherMapper.OOZIE_JOB_ID, jobId);
- actionConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId);
+ actionConf.set(LauncherAMUtils.OOZIE_JOB_ID, jobId);
+ actionConf.set(LauncherAMUtils.OOZIE_ACTION_ID, actionId);
if (Services.get().getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
List<String> purgedEntries = new ArrayList<String>();
@@ -171,9 +169,9 @@ public class LauncherHelper {
if (succeeded) {
Counters counters = runningJob.getCounters();
if (counters != null) {
- Counters.Group group = counters.getGroup(LauncherMapper.COUNTER_GROUP);
+ Counters.Group group = counters.getGroup(LauncherAMUtils.COUNTER_GROUP);
if (group != null) {
- succeeded = group.getCounter(LauncherMapper.COUNTER_LAUNCHER_ERROR) == 0;
+ succeeded = group.getCounter(LauncherAMUtils.COUNTER_LAUNCHER_ERROR) == 0;
}
}
}
@@ -187,7 +185,7 @@ public class LauncherHelper {
* @throws IOException
*/
public static boolean hasExternalChildJobs(Map<String, String> actionData) throws IOException {
- return actionData.containsKey(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS);
+ return actionData.containsKey(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS);
}
/**
@@ -197,7 +195,7 @@ public class LauncherHelper {
* @throws IOException
*/
public static boolean hasOutputData(Map<String, String> actionData) throws IOException {
- return actionData.containsKey(LauncherMapper.ACTION_DATA_OUTPUT_PROPS);
+ return actionData.containsKey(LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS);
}
/**
@@ -207,7 +205,7 @@ public class LauncherHelper {
* @throws IOException
*/
public static boolean hasStatsData(Map<String, String> actionData) throws IOException{
- return actionData.containsKey(LauncherMapper.ACTION_DATA_STATS);
+ return actionData.containsKey(LauncherAMUtils.ACTION_DATA_STATS);
}
/**
@@ -217,7 +215,7 @@ public class LauncherHelper {
* @throws IOException
*/
public static boolean hasIdSwap(Map<String, String> actionData) throws IOException {
- return actionData.containsKey(LauncherMapper.ACTION_DATA_NEW_ID);
+ return actionData.containsKey(LauncherAMUtils.ACTION_DATA_NEW_ID);
}
/**
@@ -226,7 +224,7 @@ public class LauncherHelper {
* @return
*/
public static Path getActionDataSequenceFilePath(Path actionDir) {
- return new Path(actionDir, LauncherMapper.ACTION_DATA_SEQUENCE_FILE);
+ return new Path(actionDir, LauncherAMUtils.ACTION_DATA_SEQUENCE_FILE);
}
/**
@@ -269,35 +267,35 @@ public class LauncherHelper {
if (file.equals(new Path(actionDir, "externalChildIds.properties"))) {
is = fs.open(file);
reader = new BufferedReader(new InputStreamReader(is));
- ret.put(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS,
+ ret.put(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS,
IOUtils.getReaderAsString(reader, -1));
}
else if (file.equals(new Path(actionDir, "newId.properties"))) {
is = fs.open(file);
reader = new BufferedReader(new InputStreamReader(is));
props = PropertiesUtils.readProperties(reader, -1);
- ret.put(LauncherMapper.ACTION_DATA_NEW_ID, props.getProperty("id"));
+ ret.put(LauncherAMUtils.ACTION_DATA_NEW_ID, props.getProperty("id"));
}
- else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_OUTPUT_PROPS))) {
- int maxOutputData = conf.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA,
+ else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS))) {
+ int maxOutputData = conf.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA,
2 * 1024);
is = fs.open(file);
reader = new BufferedReader(new InputStreamReader(is));
- ret.put(LauncherMapper.ACTION_DATA_OUTPUT_PROPS, PropertiesUtils
+ ret.put(LauncherAMUtils.ACTION_DATA_OUTPUT_PROPS, PropertiesUtils
.propertiesToString(PropertiesUtils.readProperties(reader, maxOutputData)));
}
- else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_STATS))) {
- int statsMaxOutputData = conf.getInt(LauncherMapper.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE,
+ else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_STATS))) {
+ int statsMaxOutputData = conf.getInt(LauncherAMUtils.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE,
Integer.MAX_VALUE);
is = fs.open(file);
reader = new BufferedReader(new InputStreamReader(is));
- ret.put(LauncherMapper.ACTION_DATA_STATS, PropertiesUtils
+ ret.put(LauncherAMUtils.ACTION_DATA_STATS, PropertiesUtils
.propertiesToString(PropertiesUtils.readProperties(reader, statsMaxOutputData)));
}
- else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_ERROR_PROPS))) {
+ else if (file.equals(new Path(actionDir, LauncherAMUtils.ACTION_DATA_ERROR_PROPS))) {
is = fs.open(file);
reader = new BufferedReader(new InputStreamReader(is));
- ret.put(LauncherMapper.ACTION_DATA_ERROR_PROPS, IOUtils.getReaderAsString(reader, -1));
+ ret.put(LauncherAMUtils.ACTION_DATA_ERROR_PROPS, IOUtils.getReaderAsString(reader, -1));
}
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java
deleted file mode 100644
index 4dba776..0000000
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.oozie.service.ConfigurationService;
-import org.apache.oozie.util.XLog;
-
-class LauncherInputFormatClassLocator {
- private final XLog LOG = XLog.getLog(getClass());
-
- static final String HADOOP_INPUT_FORMAT_CLASSNAME = "mapreduce.input.format.class";
-
- private Class<? extends InputFormat> launcherInputFormatClass;
-
- /**
- * Get the {@code LauncherMapper} map only MR job's {@link org.apache.hadoop.mapreduce.InputFormat} according to configuration.
- * <p/>
- * Priority:
- * <ol>
- * <li><tt>oozie.action.launcher.mapreduce.input-format.class-name</tt></li>
- * <li><tt>OozieLauncherInputFormat</tt></li>
- * </ol>
- *
- * @return
- */
- Class<? extends InputFormat> locateOrGet() {
- if (launcherInputFormatClass == null) {
- launcherInputFormatClass = locate();
- }
-
- return launcherInputFormatClass;
- }
-
- /**
- * Get the {@code LauncherMapper} map only MR job's {@link org.apache.hadoop.mapreduce.InputFormat} according to configuration.
- * <p/>
- * Priority:
- * <ol>
- * <li><tt>oozie.action.launcher.mapreduce.input-format.class-name</tt></li>
- * <li><tt>OozieLauncherInputFormat</tt></li>
- * </ol>
- *
- * @return
- */
- private Class<? extends InputFormat> locate() {
- String inputFormatClassName = OozieLauncherInputFormat.class.getSimpleName();
-
- final String configuredClassName = ConfigurationService.get(
- JavaActionExecutor.OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_INPUT_FORMAT_CLASSNAME);
- if (configuredClassName != null) {
- inputFormatClassName = configuredClassName;
- }
-
- try {
- LOG.debug("Locating launcher input format class [{0}].", inputFormatClassName);
-
- final Class inputFormatClass = Class.forName(inputFormatClassName);
- LOG.debug("Launcher input format class [{0}] located successfully.", inputFormatClassName);
-
- return inputFormatClass;
- } catch (final ClassNotFoundException | ClassCastException e) {
- LOG.warn("Could not load class [{0}], located [{1}] instead.", inputFormatClassName,
- OozieLauncherInputFormat.class.getSimpleName());
- return OozieLauncherInputFormat.class;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
index 634a1cb..4427688 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceActionExecutor.java
@@ -25,8 +25,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import javax.annotation.Nonnull;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -35,14 +33,6 @@ import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.oozie.action.ActionExecutor;
import org.apache.oozie.action.ActionExecutorException;
import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.ConfigurationService;
@@ -52,11 +42,6 @@ import org.apache.oozie.util.XmlUtils;
import org.jdom.Element;
import org.jdom.Namespace;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Iterables;
-import com.google.common.io.Closeables;
-
public class MapReduceActionExecutor extends JavaActionExecutor {
public static final String OOZIE_ACTION_EXTERNAL_STATS_WRITE = "oozie.action.external.stats.write";
@@ -99,14 +84,14 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
String mainClass;
Namespace ns = actionXml.getNamespace();
if (actionXml.getChild("streaming", ns) != null) {
- mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, STREAMING_MAIN_CLASS_NAME);
+ mainClass = launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, STREAMING_MAIN_CLASS_NAME);
}
else {
if (actionXml.getChild("pipes", ns) != null) {
- mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName());
+ mainClass = launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName());
}
else {
- mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName());
+ mainClass = launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName());
}
}
return mainClass;
@@ -125,7 +110,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
// Inject config-class for launcher to use for action
Element e = actionXml.getChild("config-class", actionXml.getNamespace());
if (e != null) {
- conf.set(LauncherMapper.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim());
+ conf.set(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim());
}
}
@@ -363,7 +348,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
throw convertException(e);
}
- final String newId = actionData.get(LauncherMapper.ACTION_DATA_NEW_ID);
+ final String newId = actionData.get(LauncherAMUtils.ACTION_DATA_NEW_ID);
// check the Hadoop job if newID is defined (which should be the case here) - otherwise perform the normal check()
if (newId != null) {
@@ -403,7 +388,7 @@ public class MapReduceActionExecutor extends JavaActionExecutor {
}
// run original check() if the MR action is completed or there are errors - otherwise mark it as RUNNING
- if (jobCompleted || actionData.containsKey(LauncherMapper.ACTION_DATA_ERROR_PROPS)) {
+ if (jobCompleted || actionData.containsKey(LauncherAMUtils.ACTION_DATA_ERROR_PROPS)) {
super.check(context, action);
} else {
context.setExternalStatus(RUNNING);
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
index 8a24ac3..6541d52 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/PigActionExecutor.java
@@ -60,7 +60,7 @@ public class PigActionExecutor extends ScriptLanguageActionExecutor {
@Override
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
- return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PIG_MAIN_CLASS_NAME);
+ return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, PIG_MAIN_CLASS_NAME);
}
@Override
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
index d44bbc5..9591cd9 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/ShellActionExecutor.java
@@ -44,7 +44,7 @@ public class ShellActionExecutor extends JavaActionExecutor {
@Override
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
- return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, ShellMain.class.getName());
+ return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, ShellMain.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
index 00497a7..80d64ec 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SparkActionExecutor.java
@@ -19,10 +19,8 @@
package org.apache.oozie.action.hadoop;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.action.ActionExecutorException;
-import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.service.ConfigurationService;
import org.apache.oozie.service.Services;
import org.apache.oozie.service.SparkConfigurationService;
@@ -154,7 +152,7 @@ public class SparkActionExecutor extends JavaActionExecutor {
@Override
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
- return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME);
+ return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, SPARK_MAIN_CLASS_NAME);
}
@Override
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
index 955f3b7..8fdc50c 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.oozie.action.ActionExecutorException;
@@ -64,7 +63,7 @@ public class SqoopActionExecutor extends JavaActionExecutor {
@Override
protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
- return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SQOOP_MAIN_CLASS_NAME);
+ return launcherConf.get(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, SQOOP_MAIN_CLASS_NAME);
}
@Override
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index c60a458..5629a89 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1804,7 +1804,7 @@ will be the requeue interval for the actions which are waiting for a long time w
</description>
</property>
- <!-- LauncherMapper -->
+ <!-- LauncherAMUtils -->
<property>
<name>oozie.action.max.output.data</name>
<value>2048</value>
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java b/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java
index 16ab729..6cee7a8 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java
@@ -69,7 +69,7 @@ public class LauncherMainTester {
}
// by using NULL as permission, if an underlaying SecurityManager is in place
// a security exception will be thrown. As there is not underlaying SecurityManager
- // this tests that the delegation logic of the LauncherMapper SecurityManager is
+ // this tests that the delegation logic of the LauncherAMUtils SecurityManager is
// correct for both checkPermission() signatures.
sm.checkPermission(null);
sm.checkPermission(null, sm.getSecurityContext());
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
index 199962e..3f000d5 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestDistcpMain.java
@@ -31,8 +31,6 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
-import org.apache.oozie.action.hadoop.DistcpMain;
-
public class TestDistcpMain extends MainTestCase {
@Override
@@ -49,7 +47,7 @@ public class TestDistcpMain extends MainTestCase {
writer.close();
Path outputDir = new Path(getFsTestCaseDir(), "output");
- jobConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, "org.apache.hadoop.tools.DistCp");
+ jobConf.set(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS, "org.apache.hadoop.tools.DistCp");
jobConf.set("mapreduce.job.tags", "" + System.currentTimeMillis());
setSystemProperty("oozie.job.launch.time", "" + System.currentTimeMillis());
@@ -75,7 +73,7 @@ public class TestDistcpMain extends MainTestCase {
DistcpMain.main(new String[]{inputDir.toString(), outputDir.toString()});
assertTrue(getFileSystem().exists(outputDir));
assertTrue(hadoopIdsFile.exists());
- assertNotNull(LauncherMapper.getLocalFileContentStr(hadoopIdsFile, "", -1));
+ assertNotNull(LauncherAMUtils.getLocalFileContentStr(hadoopIdsFile, "", -1));
fs.delete(outputDir,true);
// Check exception handling
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 67c7d06..59a21c4 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -264,7 +264,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals("org.apache.oozie.action.hadoop.JavaMain", ae.getLauncherMain(conf, actionXml));
assertTrue(conf.get("mapred.child.java.opts").contains("JAVA-OPTS"));
assertTrue(conf.get("mapreduce.map.java.opts").contains("JAVA-OPTS"));
- assertEquals(Arrays.asList("A1", "A2"), Arrays.asList(LauncherMapper.getMainArguments(conf)));
+ assertEquals(Arrays.asList("A1", "A2"), Arrays.asList(LauncherAMUtils.getMainArguments(conf)));
actionXml = XmlUtils.parseXml("<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
"<name-node>" + getNameNodeUri() + "</name-node> <configuration>" +
@@ -2177,8 +2177,8 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
}
public void testSetRootLoggerLevel() throws Exception {
- String oozieActionRootLogger = "oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL;
- String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherMapper.ROOT_LOGGER_LEVEL;
+ String oozieActionRootLogger = "oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL;
+ String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherAMUtils.ROOT_LOGGER_LEVEL;
// default should be INFO
JavaActionExecutor jae = new JavaActionExecutor();
@@ -2211,7 +2211,7 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
private void testEmptyArgs(boolean nullArgsAllowed, String expectedExternalStatus, WorkflowAction.Status expectedStatus)
throws Exception {
- ConfigurationService.setBoolean(LauncherMapper.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed);
+ ConfigurationService.setBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed);
String actionXml = "<java>" +
"<job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
index 1b7724e..a3b3b7b 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncher.java
@@ -38,12 +38,6 @@ import java.io.Writer;
import java.net.URI;
import java.util.Map;
-// TODO
-// this whole class can be deleted - for now, just renamed the tests that fail
-// These tests mostly validate LaunhcherMapper - with OOYA, LauncherMapper should be eliminated, too
-
-// With Hadoop 2.4.0, things work slightly differently (there is an exception in LauncherMapper.map()), also,
-// SequenceFile.Reader got deprecated
public class TestLauncher extends XFsTestCase {
@Override
@@ -58,245 +52,6 @@ public class TestLauncher extends XFsTestCase {
super.tearDown();
}
- private RunningJob _test(String... arg) throws Exception {
- Path actionDir = getFsTestCaseDir();
-
- File jar = IOUtils.createJar(new File(getTestCaseDir()), "launcher.jar", LauncherMapper.class,
- LauncherMainException.class,
- LauncherSecurityManager.class, LauncherException.class, LauncherMainTester.class);
-
- FileSystem fs = getFileSystem();
-
- Path launcherJar = new Path(actionDir, "launcher.jar");
- fs.copyFromLocalFile(new Path(jar.toString()), launcherJar);
-
- Configuration appConf = Services.get().get(HadoopAccessorService.class).
- createConfiguration(new URI(getNameNodeUri()).getAuthority());
- appConf.set("user.name", getTestUser());
- appConf.setInt("mapred.map.tasks", 1);
- appConf.setInt("mapred.map.max.attempts", 1);
- appConf.setInt("mapred.reduce.max.attempts", 1);
-
- appConf.set("mapreduce.framework.name", "yarn");
- appConf.set("mapred.job.tracker", getJobTrackerUri());
- appConf.set("fs.default.name", getNameNodeUri());
-
-
- LauncherHelper.setupMainClass(appConf, LauncherMainTester.class.getName());
- LauncherHelper.setupMainArguments(appConf, arg);
-
- Configuration actionConf = new XConfiguration();
- LauncherHelper.setupLauncherInfo(appConf, "1", "1@a", actionDir, "1@a-0", actionConf, "");
- LauncherHelper.setupYarnRestartHandling(appConf, appConf, "1@a", System.currentTimeMillis());
-
- assertEquals("1", actionConf.get("oozie.job.id"));
- assertEquals("1@a", actionConf.get("oozie.action.id"));
-
- DistributedCache.addFileToClassPath(new Path(launcherJar.toUri().getPath()), appConf);
-
- JobClient jobClient = createJobClient();
-
- final RunningJob runningJob = jobClient.submitJob(new JobConf(appConf));
-
- System.out.println("Action Dir: " + actionDir);
- System.out.println("LauncherMapper ID: " + runningJob.getJobID().toString());
-
- waitFor(180 * 1000, new Predicate() {
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
-
- assertTrue(appConf.get("oozie.action.prepare.xml").equals(""));
- return runningJob;
-
- }
-
- public void ___testEmpty() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- final RunningJob runningJob = _test();
- waitFor(2000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
-
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf);
- assertFalse(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir)));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertTrue(LauncherHelper.isMainSuccessful(runningJob));
- assertFalse(LauncherHelper.hasOutputData(actionData));
- assertFalse(LauncherHelper.hasIdSwap(actionData));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- }
-
- public void ___testExit0() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- final RunningJob runningJob = _test("exit0");
- waitFor(2000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
-
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf);
- assertFalse(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir)));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertTrue(LauncherHelper.isMainSuccessful(runningJob));
- assertFalse(LauncherHelper.hasOutputData(actionData));
- assertFalse(LauncherHelper.hasIdSwap(actionData));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- }
-
- public void ___testExit1() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- final RunningJob runningJob = _test("exit1");
- waitFor(2000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
-
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf);
- assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir)));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertFalse(LauncherHelper.isMainSuccessful(runningJob));
- assertFalse(LauncherHelper.hasOutputData(actionData));
- assertFalse(LauncherHelper.hasIdSwap(actionData));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertTrue(actionData.containsKey(LauncherMapper.ACTION_DATA_ERROR_PROPS));
- }
-
- public void ___testException() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- final RunningJob runningJob = _test("exception");
- waitFor(2000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
-
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf);
- assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir)));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertFalse(LauncherHelper.isMainSuccessful(runningJob));
- assertFalse(LauncherHelper.hasOutputData(actionData));
- assertFalse(LauncherHelper.hasIdSwap(actionData));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- }
-
- public void __testThrowable() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- final RunningJob runningJob = _test("throwable");
- waitFor(2000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
-
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf);
- assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir)));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertFalse(LauncherHelper.isMainSuccessful(runningJob));
- assertFalse(LauncherHelper.hasOutputData(actionData));
- assertFalse(LauncherHelper.hasIdSwap(actionData));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- }
-
- public void __testOutput() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- final RunningJob runningJob = _test("out");
- waitFor(2000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
-
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf);
- assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir)));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertTrue(LauncherHelper.isMainSuccessful(runningJob));
- assertTrue(LauncherHelper.hasOutputData(actionData));
- assertFalse(LauncherHelper.hasIdSwap(actionData));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- }
-
- public void __testNewId() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- final RunningJob runningJob = _test("id");
- waitFor(2000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
-
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf);
- assertTrue(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir)));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertTrue(LauncherHelper.isMainSuccessful(runningJob));
- assertFalse(LauncherHelper.hasOutputData(actionData));
- assertTrue(LauncherHelper.hasIdSwap(actionData));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- }
-
- public void __testSecurityManager() throws Exception {
- Path actionDir = getFsTestCaseDir();
- FileSystem fs = getFileSystem();
- final RunningJob runningJob = _test("securityManager");
- waitFor(2000, new Predicate() {
- @Override
- public boolean evaluate() throws Exception {
- return runningJob.isComplete();
- }
- });
- assertTrue(runningJob.isSuccessful());
-
- Configuration conf = new XConfiguration();
- conf.set("user.name", getTestUser());
- Map<String, String> actionData = LauncherHelper.getActionData(fs, actionDir, conf);
- assertFalse(fs.exists(LauncherHelper.getActionDataSequenceFilePath(actionDir)));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- assertTrue(LauncherHelper.isMainSuccessful(runningJob));
- assertFalse(LauncherHelper.hasOutputData(actionData));
- assertFalse(LauncherHelper.hasIdSwap(actionData));
- assertTrue(LauncherHelper.isMainDone(runningJob));
- }
-
// Test to ensure that the property value "oozie.action.prepare.xml" in the configuration of the job is an empty
// string when there is no prepare block in workflow XML or there is one with no prepare actions in it
public void testSetupLauncherInfoWithEmptyPrepareXML() throws Exception {
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java
deleted file mode 100644
index a6ccc5a..0000000
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.oozie.action.hadoop;
-
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.oozie.service.ConfigurationService;
-
-public class TestLauncherInputFormatClassLocator extends ActionExecutorTestCase {
-
- public void testInputFormatClassnameNotSet() {
- ConfigurationService.set(
- "oozie.action.launcher.mapreduce.input.format.class",
- "");
-
- final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet();
-
- assertEquals("default launcher input format is found when configuration is left empty",
- OozieLauncherInputFormat.class,
- inputFormatClass);
- }
-
- public void testInputFormatClassnameDefault() {
- final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet();
-
- assertEquals("default launcher input format is found when configuration is left on default",
- OozieLauncherInputFormat.class,
- inputFormatClass);
- }
-
- public void testInputFormatClassnameSet() {
- ConfigurationService.set(
- "oozie.action.launcher.mapreduce.input.format.class",
- InputFormat.class.getName());
-
- final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet();
-
- assertEquals("Hadoop launcher input format is found when configuration is set",
- InputFormat.class,
- inputFormatClass);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java
index a4b9943..8c716e8 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceMain.java
@@ -70,7 +70,7 @@ public class TestMapReduceMain extends MainTestCase {
MapReduceMain.main(new String[0]);
assertTrue(newId.exists());
- assertNotNull(LauncherMapper.getLocalFileContentStr(newId, "", -1));
+ assertNotNull(LauncherAMUtils.getLocalFileContentStr(newId, "", -1));
return null;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
index 72be0a2..8d0b441 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestShellActionExecutor.java
@@ -268,7 +268,7 @@ public class TestShellActionExecutor extends ActionExecutorTestCase {
public void testEnvVar() throws Exception {
Services.get().destroy();
Services services = new Services();
- services.getConf().setInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 8 * 1042);
+ services.getConf().setInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, 8 * 1042);
services.init();
FileSystem fs = getFileSystem();
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
index 7bc81c8..4cb2530 100644
--- a/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
+++ b/core/src/test/java/org/apache/oozie/service/TestConfigurationService.java
@@ -21,8 +21,7 @@ package org.apache.oozie.service;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.action.hadoop.CredentialsProviderFactory;
import org.apache.oozie.action.hadoop.DistcpActionExecutor;
-import org.apache.oozie.action.hadoop.JavaActionExecutor;
-import org.apache.oozie.action.hadoop.LauncherMapper;
+import org.apache.oozie.action.hadoop.LauncherAMUtils;
import org.apache.oozie.command.coord.CoordActionInputCheckXCommand;
import org.apache.oozie.command.coord.CoordSubmitXCommand;
import org.apache.oozie.command.wf.JobXCommand;
@@ -210,7 +209,7 @@ public class TestConfigurationService extends XTestCase {
assertEquals("300000", ConfigurationService.get(JPAService.CONF_VALIDATE_DB_CONN_EVICTION_INTERVAL).trim());
assertEquals("10", ConfigurationService.get(JPAService.CONF_VALIDATE_DB_CONN_EVICTION_NUM).trim());
- assertEquals(2048, ConfigurationService.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA));
+ assertEquals(2048, ConfigurationService.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA));
assertEquals("http://0.0.0.0:11000/oozie?job=", ConfigurationService.get(JobXCommand.CONF_CONSOLE_URL));
assertEquals(false, ConfigurationService.getBoolean(HadoopAccessorService.KERBEROS_AUTH_ENABLED));
@@ -245,7 +244,7 @@ public class TestConfigurationService extends XTestCase {
assertEquals("org.apache.oozie.dependency.FSURIHandler",
ConfigurationService.getStrings(URIHandlerService.URI_HANDLERS)[0]);
assertEquals(cl.getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false),
- ConfigurationService.getBoolean(LauncherMapper.HADOOP2_WORKAROUND_DISTRIBUTED_CACHE));
+ ConfigurationService.getBoolean(LauncherAMUtils.HADOOP2_WORKAROUND_DISTRIBUTED_CACHE));
assertEquals("org.apache.oozie.event.MemoryEventQueue",
(ConfigurationService.getClass(cl.getConf(), EventHandlerService.CONF_EVENT_QUEUE).getName()));
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 3cd95af..de6aff5 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.0.0 release (trunk - unreleased)
+OOZIE-2918 Delete LauncherMapper and its test (asasvari via pbacsko)
OOZIE-2733 change org.apache.hadoop.fs.permission.AccessControlException to org.apache.hadoop.security.AccessControlException (gezapeti)
OOZIE-2884 consolidate hadoop versions in pomfiles (Artem Ervits via gezapeti)
OOZIE-2794 Hive and Pig tests failing when multiple pre-commit jobs are running (gezapeti)
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
----------------------------------------------------------------------
diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
index d7cf9d9..d17d5be 100644
--- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
+++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java
@@ -55,7 +55,7 @@ public class DistcpMain extends JavaMain {
Configuration actionConf = loadActionConf();
LauncherMain.killChildYarnJobs(actionConf);
String logFile = setUpDistcpLog4J(actionConf);
- Class<?> klass = actionConf.getClass(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS,
+ Class<?> klass = actionConf.getClass(LauncherAMUtils.CONF_OOZIE_ACTION_MAIN_CLASS,
org.apache.hadoop.tools.DistCp.class);
System.out.println("Main class : " + klass.getName());
System.out.println("Arguments :");
@@ -125,7 +125,7 @@ public class DistcpMain extends JavaMain {
String logFile = new File("distcp-oozie-" + hadoopJobId + ".log").getAbsolutePath();
String logLevel = distcpConf.get("oozie.distcp.log.level", "INFO");
- String rootLogLevel = distcpConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ String rootLogLevel = distcpConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO");
log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A");
log4jProperties.setProperty("log4j.logger.org.apache.hadoop.tools", logLevel + ", A, jobid");
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index 7b152f2..fb3fabf 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -136,7 +136,7 @@ public class HiveMain extends LauncherMain {
String logFile = new File("hive-oozie-" + hadoopJobId + ".log").getAbsolutePath();
String logLevel = hiveConf.get("oozie.hive.log.level", "INFO");
- String rootLogLevel = hiveConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ String rootLogLevel = hiveConf.get("oozie.action." + LauncherAMUtils.ROOT_LOGGER_LEVEL, "INFO");
log4jProperties.setProperty("log4j.rootLogger", rootLogLevel + ", A");
log4jProperties.setProperty("log4j.logger.org.apache.hadoop.hive", logLevel + ", A");
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
index 71ee641..0e99fac 100644
--- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
+++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveActionExecutor.java
@@ -170,7 +170,7 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase {
assertNotNull(context.getExternalChildIDs());
//while this works in a real cluster, it does not with miniMR
//assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0);
- //assertTrue(!actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty());
+ //assertTrue(!actionData.get(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty());
assertTrue(fs.exists(outputDir));
assertTrue(fs.isDirectory(outputDir));
}
@@ -193,7 +193,7 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase {
assertNotNull(context.getAction().getExternalChildIDs());
//while this works in a real cluster, it does not with miniMR
//assertTrue(outputData.getProperty(LauncherMain.HADOOP_JOBS).trim().length() > 0);
- //assertTrue(!actionData.get(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty());
+ //assertTrue(!actionData.get(LauncherAMUtils.ACTION_DATA_EXTERNAL_CHILD_IDS).isEmpty());
assertTrue(fs.exists(outputDir));
assertTrue(fs.isDirectory(outputDir));
}
@@ -266,7 +266,7 @@ public class TestHiveActionExecutor extends ActionExecutorTestCase {
Context context = createContext(getActionScriptXml());
Namespace ns = Namespace.getNamespace("uri:oozie:hive-action:0.2");
submitAction(context, ns);
- FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherMapper.ACTION_CONF_XML));
+ FSDataInputStream os = fs.open(new Path(context.getActionDir(), LauncherAMUtils.ACTION_CONF_XML));
XConfiguration conf = new XConfiguration();
conf.addResource(os);
assertNull(conf.get("oozie.HadoopAccessorService.created"));
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
index bbd6246..35c2865 100644
--- a/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
+++ b/sharelib/hive/src/test/java/org/apache/oozie/action/hadoop/TestHiveMain.java
@@ -135,7 +135,8 @@ public class TestHiveMain extends MainTestCase {
setSystemProperty("oozie.action.conf.xml", actionXml.getAbsolutePath());
setSystemProperty("oozie.action.externalChildIDs", externalChildIdsFile.getAbsolutePath());
- new LauncherSecurityManager();
+ LauncherAM.LauncherSecurityManager launcherSecurityManager = new LauncherAM.LauncherSecurityManager();
+ launcherSecurityManager.enable();
String user = System.getProperty("user.name");
try {
os = new FileOutputStream(hiveSite);
@@ -145,10 +146,10 @@ public class TestHiveMain extends MainTestCase {
HiveMain.main(null);
}
catch (SecurityException ex) {
- if (LauncherSecurityManager.getExitInvoked()) {
- System.out.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")");
- System.err.println("Intercepting System.exit(" + LauncherSecurityManager.getExitCode() + ")");
- if (LauncherSecurityManager.getExitCode() != 0) {
+ if (launcherSecurityManager.getExitInvoked()) {
+ System.out.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")");
+ System.err.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")");
+ if (launcherSecurityManager.getExitCode() != 0) {
fail();
}
}
@@ -163,7 +164,7 @@ public class TestHiveMain extends MainTestCase {
}
assertTrue(externalChildIdsFile.exists());
- assertNotNull(LauncherMapper.getLocalFileContentStr(externalChildIdsFile, "", -1));
+ assertNotNull(LauncherAMUtils.getLocalFileContentStr(externalChildIdsFile, "", -1));
//TODO: I cannot figure out why when log file is not created in this testcase, it works when running in Launcher
// Properties props = new Properties();
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java
index 512730a..fd6c283 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/FSLauncherURIHandler.java
@@ -58,7 +58,7 @@ public class FSLauncherURIHandler implements LauncherURIHandler {
FileSystem fs = FileSystem.get(uri, conf);
Path[] pathArr = FileUtil.stat2Paths(fs.globStatus(getNormalizedPath(uri)));
if (pathArr != null && pathArr.length > 0) {
- int fsGlobMax = conf.getInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX, 1000);
+ int fsGlobMax = conf.getInt(LauncherAMUtils.CONF_OOZIE_ACTION_FS_GLOB_MAX, 1000);
if (pathArr.length > fsGlobMax) {
throw new LauncherException("exceeds max number (" + fsGlobMax
+ ") of files/dirs to delete in <prepare>");
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
index c3e3d3f..f98c781 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java
@@ -28,7 +28,7 @@ public class JavaMain extends LauncherMain {
public static final String JAVA_MAIN_CLASS = "oozie.action.java.main";
/**
- * @param args Invoked from LauncherMapper:map()
+ * @param args Invoked from LauncherAM:run()
* @throws Exception
*/
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
index 4f252d1..bac17b2 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAM.java
@@ -537,7 +537,7 @@ public class LauncherAM {
}
private String[] getMainArguments(Configuration conf) {
- return LauncherMapper.getMainArguments(conf);
+ return LauncherAMUtils.getMainArguments(conf);
}
public static class LauncherSecurityManager extends SecurityManager {
@@ -593,6 +593,11 @@ public class LauncherAM {
System.setSecurityManager(originalSecurityManager);
}
}
+
+ public void reset() {
+ exitInvoked = false;
+ exitCode = 0;
+ }
}
public enum OozieActionResult {
@@ -611,4 +616,4 @@ public class LauncherAM {
return yarnStatus;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/c09561ff/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMUtils.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMUtils.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMUtils.java
new file mode 100644
index 0000000..74c7af0
--- /dev/null
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherAMUtils.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import com.google.common.base.Strings;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+
+public class LauncherAMUtils {
+
+ static final String CONF_OOZIE_ACTION_MAIN_CLASS = "oozie.launcher.action.main.class";
+
+ static final String ACTION_PREFIX = "oozie.action.";
+ public static final String CONF_OOZIE_ACTION_MAX_OUTPUT_DATA = ACTION_PREFIX + "max.output.data";
+ static final String CONF_OOZIE_ACTION_MAIN_ARG_COUNT = ACTION_PREFIX + "main.arg.count";
+ static final String CONF_OOZIE_ACTION_MAIN_ARG_PREFIX = ACTION_PREFIX + "main.arg.";
+ static final String CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE = "oozie.external.stats.max.size";
+ static final String OOZIE_ACTION_CONFIG_CLASS = ACTION_PREFIX + "config.class";
+ static final String CONF_OOZIE_ACTION_FS_GLOB_MAX = ACTION_PREFIX + "fs.glob.max";
+ static final String CONF_OOZIE_NULL_ARGS_ALLOWED = ACTION_PREFIX + "null.args.allowed";
+
+ static final String COUNTER_GROUP = "oozie.launcher";
+ static final String COUNTER_LAUNCHER_ERROR = "oozie.launcher.error";
+
+ static final String OOZIE_JOB_ID = "oozie.job.id";
+ static final String OOZIE_ACTION_ID = ACTION_PREFIX + "id";
+ static final String OOZIE_ACTION_RECOVERY_ID = ACTION_PREFIX + "recovery.id";
+
+ static final String OOZIE_ACTION_DIR_PATH = ACTION_PREFIX + "dir.path";
+ static final String ACTION_CONF_XML = "action.xml";
+ static final String ACTION_PREPARE_XML = "oozie.action.prepare.xml";
+ static final String ACTION_DATA_SEQUENCE_FILE = "action-data.seq"; // COMBO FILE
+ static final String ACTION_DATA_EXTERNAL_CHILD_IDS = "externalChildIDs";
+ static final String ACTION_DATA_OUTPUT_PROPS = "output.properties";
+ static final String ACTION_DATA_STATS = "stats.properties";
+ static final String ACTION_DATA_NEW_ID = "newId";
+ static final String ACTION_DATA_ERROR_PROPS = "error.properties";
+ public static final String HADOOP2_WORKAROUND_DISTRIBUTED_CACHE = "oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache";
+ public static final String PROPAGATION_CONF_XML = "propagation-conf.xml";
+ public static final String ROOT_LOGGER_LEVEL = "rootlogger.log.level";
+
+
+ public static String getLocalFileContentStr(File file, String type, int maxLen) throws LauncherException, IOException {
+ StringBuffer sb = new StringBuffer();
+ FileReader reader = new FileReader(file);
+ char[] buffer = new char[2048];
+ int read;
+ int count = 0;
+ while ((read = reader.read(buffer)) > -1) {
+ count += read;
+ if (maxLen > -1 && count > maxLen) {
+ throw new LauncherException(type + " data exceeds its limit ["+ maxLen + "]");
+ }
+ sb.append(buffer, 0, read);
+ }
+ reader.close();
+ return sb.toString();
+ }
+
+ public static String[] getMainArguments(Configuration conf) {
+ String[] args = new String[conf.getInt(CONF_OOZIE_ACTION_MAIN_ARG_COUNT, 0)];
+
+ String[] retArray;
+
+ if (conf.getBoolean(CONF_OOZIE_NULL_ARGS_ALLOWED, true)) {
+ for (int i = 0; i < args.length; i++) {
+ args[i] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ }
+
+ retArray = args;
+ } else {
+ int pos = 0;
+ for (int i = 0; i < args.length; i++) {
+ String arg = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ if (!Strings.isNullOrEmpty(arg)) {
+ args[pos++] = conf.get(CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i);
+ }
+ }
+
+ // this is to skip null args, that is <arg></arg> in the workflow XML -- in this case,
+ // args[] might look like {"arg1", "arg2", null, null} at this point
+ retArray = new String[pos];
+ System.arraycopy(args, 0, retArray, 0, pos);
+ }
+
+ return retArray;
+ }
+}
+