You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pu...@apache.org on 2016/01/21 17:58:52 UTC
oozie git commit: OOZIE-2430 Add root logger for hive,sqoop action
Repository: oozie
Updated Branches:
refs/heads/master 1600045ff -> 55171fc09
OOZIE-2430 Add root logger for hive,sqoop action
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/55171fc0
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/55171fc0
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/55171fc0
Branch: refs/heads/master
Commit: 55171fc0949c76c6c9bb8ab0be6f4f7191831207
Parents: 1600045
Author: Purshotam Shah <pu...@yahoo-inc.com>
Authored: Thu Jan 21 08:58:30 2016 -0800
Committer: Purshotam Shah <pu...@yahoo-inc.com>
Committed: Thu Jan 21 08:58:30 2016 -0800
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 41 +++++++++++++++++---
core/src/main/resources/oozie-default.xml | 8 ++++
.../action/hadoop/TestJavaActionExecutor.java | 26 +++++++++++++
release-log.txt | 1 +
.../apache/oozie/action/hadoop/HiveMain.java | 6 ++-
.../oozie/action/hadoop/LauncherMapper.java | 1 +
.../org/apache/oozie/action/hadoop/PigMain.java | 3 +-
.../oozie/action/hadoop/PigMainWithOldAPI.java | 2 +
.../apache/oozie/action/hadoop/SqoopMain.java | 2 +
9 files changed, 83 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index c7cbdae..4fc0c52 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -33,9 +33,9 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
-import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -45,12 +45,15 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AccessControlException;
+import org.apache.hadoop.hive.shims.HadoopShims;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.DiskChecker;
import org.apache.oozie.WorkflowActionBean;
import org.apache.oozie.WorkflowJobBean;
@@ -67,6 +70,7 @@ import org.apache.oozie.service.Services;
import org.apache.oozie.service.ShareLibService;
import org.apache.oozie.service.URIHandlerService;
import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.util.ELEvaluationException;
import org.apache.oozie.util.ELEvaluator;
import org.apache.oozie.util.JobUtils;
import org.apache.oozie.util.LogUtils;
@@ -74,13 +78,9 @@ import org.apache.oozie.util.PropertiesUtils;
import org.apache.oozie.util.XConfiguration;
import org.apache.oozie.util.XLog;
import org.apache.oozie.util.XmlUtils;
-import org.apache.oozie.util.ELEvaluationException;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.Namespace;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.oozie.hadoop.utils.HadoopShims;
public class JavaActionExecutor extends ActionExecutor {
@@ -521,6 +521,7 @@ public class JavaActionExecutor extends ActionExecutor {
// set cancel.delegation.token in actionConf that child job doesn't cancel delegation token
actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
updateConfForJavaTmpDir(actionConf);
+ setRootLoggerLevel(actionConf);
return actionConf;
}
catch (IOException ex) {
@@ -534,6 +535,34 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
+ /**
+ * Set root log level property in actionConf
+ * @param actionConf
+ */
+ void setRootLoggerLevel(Configuration actionConf) {
+ String oozieActionTypeRootLogger = "oozie.action." + getType() + LauncherMapper.ROOT_LOGGER_LEVEL;
+ String oozieActionRootLogger = "oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL;
+
+ // check if root log level has already mentioned in action configuration
+ String rootLogLevel = actionConf.get(oozieActionTypeRootLogger, actionConf.get(oozieActionRootLogger));
+ if (rootLogLevel != null) {
+ // root log level is mentioned in action configuration
+ return;
+ }
+
+ // set the root log level which is mentioned in oozie default
+ rootLogLevel = ConfigurationService.get(oozieActionTypeRootLogger);
+ if (rootLogLevel != null && rootLogLevel.length() > 0) {
+ actionConf.set(oozieActionRootLogger, rootLogLevel);
+ }
+ else {
+ rootLogLevel = ConfigurationService.get(oozieActionRootLogger);
+ if (rootLogLevel != null && rootLogLevel.length() > 0) {
+ actionConf.set(oozieActionRootLogger, rootLogLevel);
+ }
+ }
+ }
+
Configuration addToCache(Configuration conf, Path appPath, String filePath, boolean archive)
throws ActionExecutorException {
@@ -1673,10 +1702,12 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
+ @Override
public boolean requiresNameNodeJobTracker() {
return true;
}
+ @Override
public boolean supportsConfigurationJobXML() {
return true;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index faf3740..ca49fa6 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1804,6 +1804,14 @@
</description>
</property>
+ <property>
+ <name>oozie.action.rootlogger.log.level</name>
+ <value>INFO</value>
+ <description>
+ Logging level for root logger
+ </description>
+ </property>
+
<!-- HadoopActionExecutor -->
<!-- This is common to the subclasses action executors for map-reduce and pig -->
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 70f33ea..86f0ed2 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -2702,4 +2702,30 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
assertEquals(4, conf.size());
}
}
+
+ public void testSetRootLoggerLevel() throws Exception {
+ String oozieActionRootLogger = "oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL;
+ String oozieActionHiveRootLogger = "oozie.action.hive" + LauncherMapper.ROOT_LOGGER_LEVEL;
+
+ // default should be INFO
+ JavaActionExecutor jae = new JavaActionExecutor();
+ Configuration conf = new Configuration(false);
+ jae.setRootLoggerLevel(conf);
+ assertEquals("INFO", conf.get(oozieActionRootLogger));
+
+ // explicitly set root logger level to DEBUG
+ jae = new JavaActionExecutor();
+ conf = new Configuration(false);
+ conf.set(oozieActionRootLogger, "DEBUG");
+ jae.setRootLoggerLevel(conf);
+ assertEquals("DEBUG", conf.get(oozieActionRootLogger));
+
+ // explicitly set root logger level to DEBUG for hive action
+ HiveActionExecutor hae = new HiveActionExecutor();
+ conf = new Configuration(false);
+ conf.set(oozieActionHiveRootLogger, "DEBUG");
+ hae.setRootLoggerLevel(conf);
+ assertEquals("DEBUG", conf.get(oozieActionHiveRootLogger));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 391834b..0804939 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.3.0 release (trunk - unreleased)
+OOZIE-2430 Add root logger for hive,sqoop action (satishsaley via puru)
OOZIE-2441 SubWorkflow action with propagate-configuration but no global section throws NPE on submit (rkanter)
OOZIE-2370 Docs for Coordinator Action Status Notification has wrong property name (eeeva via rkanter)
OOZIE-2419 HBase credentials are not correctly proxied (qwertymaniac via rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
----------------------------------------------------------------------
diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
index 94e586b..cb0172d 100644
--- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
+++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java
@@ -143,7 +143,9 @@ public class HiveMain extends LauncherMain {
}
String logLevel = hiveConf.get("oozie.hive.log.level", "INFO");
+ String rootLogLevel = hiveConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A");
hadoopProps.setProperty("log4j.logger.org.apache.hadoop.hive", logLevel + ", A");
hadoopProps.setProperty("log4j.logger.hive", logLevel + ", A");
hadoopProps.setProperty("log4j.logger.DataNucleus", logLevel + ", A");
@@ -151,13 +153,14 @@ public class HiveMain extends LauncherMain {
hadoopProps.setProperty("log4j.logger.JPOX", logLevel + ", A");
hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender");
hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout");
- hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
+ hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] -5p %c %x - %m%n");
hadoopProps.setProperty("log4j.appender.jobid", "org.apache.log4j.FileAppender");
hadoopProps.setProperty("log4j.appender.jobid.file", logFile);
hadoopProps.setProperty("log4j.appender.jobid.layout", "org.apache.log4j.PatternLayout");
hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
hadoopProps.setProperty("log4j.logger.org.apache.hadoop.hive.ql.exec", "INFO, jobid");
+ hadoopProps.setProperty("log4j.logger.SessionState", "INFO, jobid");
String localProps = new File(HIVE_L4J_PROPS).getAbsolutePath();
OutputStream os1 = new FileOutputStream(localProps);
@@ -199,6 +202,7 @@ public class HiveMain extends LauncherMain {
return hiveConf;
}
+ @Override
protected void run(String[] args) throws Exception {
System.out.println();
System.out.println("Oozie Hive action configuration");
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
index c45073f..545f8e2 100644
--- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java
@@ -83,6 +83,7 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R
public static final String HADOOP2_WORKAROUND_DISTRIBUTED_CACHE = "oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache";
public static final String PROPAGATION_CONF_XML = "propagation-conf.xml";
public static final String OOZIE_LAUNCHER_JOB_ID = "oozie.launcher.job.id";
+ public static final String ROOT_LOGGER_LEVEL = "rootlogger.log.level";
private void setRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId) throws LauncherException {
try {
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
index 8228e88..7ba22f1 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java
@@ -165,11 +165,12 @@ public class PigMain extends LauncherMain {
if (log4jFile != null) {
String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO");
+ String rootLogLevel = actionConf.get("oozie.action ." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
// append required PIG properties to the default hadoop log4j file
Properties hadoopProps = new Properties();
hadoopProps.load(log4jFile.openStream());
- hadoopProps.setProperty("log4j.rootLogger", pigLogLevel + ", A, B");
+ hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A, B");
hadoopProps.setProperty("log4j.logger.org.apache.pig", pigLogLevel + ", A, B");
hadoopProps.setProperty("log4j.additivity.org.apache.pig", "false");
hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender");
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
----------------------------------------------------------------------
diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
index 607ba1b..7ff125d 100644
--- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
+++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java
@@ -153,10 +153,12 @@ public class PigMainWithOldAPI extends LauncherMain {
if (log4jFile != null) {
String pigLogLevel = actionConf.get("oozie.pig.log.level", "INFO");
+ String rootLogLevel = actionConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
// append required PIG properties to the default hadoop log4j file
Properties hadoopProps = new Properties();
hadoopProps.load(log4jFile.openStream());
+ hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A, B");
hadoopProps.setProperty("log4j.logger.org.apache.pig", pigLogLevel + ", A, B");
hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender");
hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout");
http://git-wip-us.apache.org/repos/asf/oozie/blob/55171fc0/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
----------------------------------------------------------------------
diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
index 6ba7238..59a0209 100644
--- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
+++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
@@ -120,7 +120,9 @@ public class SqoopMain extends LauncherMain {
}
String logLevel = sqoopConf.get("oozie.sqoop.log.level", "INFO");
+ String rootLogLevel = sqoopConf.get("oozie.action." + LauncherMapper.ROOT_LOGGER_LEVEL, "INFO");
+ hadoopProps.setProperty("log4j.rootLogger", rootLogLevel + ", A");
hadoopProps.setProperty("log4j.logger.org.apache.sqoop", logLevel + ", A");
hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender");
hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout");