You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/08/14 19:23:31 UTC
[2/3] hive git commit: HIVE-11304: Migrate to Log4j2 from Log4j 1.x
(Prasanth Jayachandran reviewed by Thejas Nair, Sergey Shelukhin)
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/itests/pom.xml
----------------------------------------------------------------------
diff --git a/itests/pom.xml b/itests/pom.xml
index f156cc4..acce713 100644
--- a/itests/pom.xml
+++ b/itests/pom.xml
@@ -94,7 +94,7 @@
}
mkdir -p $DOWNLOAD_DIR
download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark"
- cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
+ cp -f $HIVE_ROOT/data/conf/spark/log4j2.xml $BASE_DIR/spark/conf/
sed '/package /d' ${basedir}/${hive.path.to.root}/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java > /tmp/UDFExampleAdd.java
javac -cp ${settings.localRepository}/org/apache/hive/hive-exec/${project.version}/hive-exec-${project.version}.jar /tmp/UDFExampleAdd.java -d /tmp
jar -cf /tmp/udfexampleadd-1.0.jar -C /tmp UDFExampleAdd.class
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index b33cb58..65117c4 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -5931,7 +5931,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
// If the log4j.configuration property hasn't already been explicitly set,
// use Hive's default log4j configuration
- if (System.getProperty("log4j.configuration") == null) {
+ if (System.getProperty("log4j.configurationFile") == null) {
// NOTE: It is critical to do this here so that log4j is reinitialized
// before any of the other core hive classes are loaded
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
index ad99427..df42f1a 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
@@ -17,15 +17,11 @@
*/
package org.apache.hadoop.hive.metastore.txn;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
import java.util.ArrayList;
import java.util.Arrays;
@@ -34,7 +30,29 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-import static junit.framework.Assert.*;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
+import org.apache.hadoop.hive.metastore.api.UnlockRequest;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
/**
* Tests for TxnHandler.
@@ -43,11 +61,9 @@ public class TestCompactionTxnHandler {
private HiveConf conf = new HiveConf();
private CompactionTxnHandler txnHandler;
- static final private Log LOG = LogFactory.getLog(TestCompactionTxnHandler.class);
public TestCompactionTxnHandler() throws Exception {
TxnDbUtil.setConfValues(conf);
- LogManager.getLogger(TxnHandler.class.getName()).setLevel(Level.DEBUG);
tearDown();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index f478184..6dc0bd3 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -17,16 +17,11 @@
*/
package org.apache.hadoop.hive.metastore.txn;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.log4j.Level;
-import org.apache.log4j.LogManager;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertFalse;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.assertTrue;
+import static junit.framework.Assert.fail;
import java.sql.Connection;
import java.sql.SQLException;
@@ -36,21 +31,66 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import static junit.framework.Assert.*;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
+import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionRequest;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
+import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockLevel;
+import org.apache.hadoop.hive.metastore.api.LockRequest;
+import org.apache.hadoop.hive.metastore.api.LockResponse;
+import org.apache.hadoop.hive.metastore.api.LockState;
+import org.apache.hadoop.hive.metastore.api.LockType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
+import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
+import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
+import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
+import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
+import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
+import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
+import org.apache.hadoop.hive.metastore.api.TxnInfo;
+import org.apache.hadoop.hive.metastore.api.TxnOpenException;
+import org.apache.hadoop.hive.metastore.api.TxnState;
+import org.apache.hadoop.hive.metastore.api.UnlockRequest;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Tests for TxnHandler.
*/
public class TestTxnHandler {
static final private String CLASS_NAME = TxnHandler.class.getName();
- static final private Log LOG = LogFactory.getLog(CLASS_NAME);
+ private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
private HiveConf conf = new HiveConf();
private TxnHandler txnHandler;
public TestTxnHandler() throws Exception {
TxnDbUtil.setConfValues(conf);
- LogManager.getLogger(TxnHandler.class.getName()).setLevel(Level.DEBUG);
+ LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ Configuration conf = ctx.getConfiguration();
+ conf.getLoggerConfig(CLASS_NAME).setLevel(Level.DEBUG);
+ ctx.updateLoggers(conf);
tearDown();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/packaging/src/main/assembly/bin.xml
----------------------------------------------------------------------
diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
index 63253c5..0fa6af8 100644
--- a/packaging/src/main/assembly/bin.xml
+++ b/packaging/src/main/assembly/bin.xml
@@ -275,7 +275,7 @@
<fileMode>644</fileMode>
<includes>
<include>webhcat-default.xml</include>
- <include>webhcat-log4j.properties</include>
+ <include>webhcat-log4j2.xml</include>
</includes>
<filtered>true</filtered>
<outputDirectory>hcatalog/etc/webhcat</outputDirectory>
@@ -323,19 +323,19 @@
<files>
<file>
- <source>${project.parent.basedir}/common/src/main/resources/hive-log4j.properties</source>
+ <source>${project.parent.basedir}/common/src/main/resources/hive-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>hive-log4j.properties.template</destName>
+ <destName>hive-log4j2.xml.template</destName>
</file>
<file>
- <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j.properties</source>
+ <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>hive-exec-log4j.properties.template</destName>
+ <destName>hive-exec-log4j2.xml.template</destName>
</file>
<file>
- <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j.properties</source>
+ <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j2.xml</source>
<outputDirectory>conf</outputDirectory>
- <destName>beeline-log4j.properties.template</destName>
+ <destName>beeline-log4j2.xml.template</destName>
</file>
<file>
<source>${project.parent.basedir}/hcatalog/README.txt</source>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0383e01..15c2805 100644
--- a/pom.xml
+++ b/pom.xml
@@ -148,8 +148,7 @@
<kryo.version>2.22</kryo.version>
<libfb303.version>0.9.2</libfb303.version>
<libthrift.version>0.9.2</libthrift.version>
- <log4j.version>1.2.16</log4j.version>
- <log4j-extras.version>1.2.17</log4j-extras.version>
+ <log4j2.version>2.3</log4j2.version>
<opencsv.version>2.3</opencsv.version>
<mockito-all.version>1.9.5</mockito-all.version>
<mina.version>2.0.0-M5</mina.version>
@@ -366,14 +365,24 @@
<version>${junit.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>apache-log4j-extras</artifactId>
- <version>${log4j-extras.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-web</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
@@ -584,11 +593,6 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
<version>${xerces.version}</version>
@@ -604,11 +608,6 @@
<artifactId>slf4j-api</artifactId>
<version>${slf4j.version}</version>
</dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
</dependencies>
<build>
@@ -872,7 +871,7 @@
<!-- required for hive-exec jar path and tests which reference a jar -->
<maven.local.repository>${maven.repo.local}</maven.local.repository>
<mapred.job.tracker>local</mapred.job.tracker>
- <log4j.configuration>${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j.properties</log4j.configuration>
+ <log4j.configurationFile>${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j2.xml</log4j.configurationFile>
<log4j.debug>true</log4j.debug>
<!-- don't diry up /tmp -->
<java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/pom.xml
----------------------------------------------------------------------
diff --git a/ql/pom.xml b/ql/pom.xml
index e7a8e7b..36b3433 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -112,14 +112,19 @@
<version>${javolution.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>apache-log4j-extras</artifactId>
- <version>${log4j-extras.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
index a2cf712..82345ee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
@@ -27,7 +27,6 @@ import java.lang.management.MemoryMXBean;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Enumeration;
import java.util.List;
import java.util.Properties;
@@ -57,13 +56,14 @@ import org.apache.hadoop.hive.ql.exec.PartitionKeySampler;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
+import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveKey;
import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
import org.apache.hadoop.hive.ql.io.IOPrepareCache;
+import org.apache.hadoop.hive.ql.log.NullAppender;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.MapWork;
@@ -88,11 +88,12 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Partitioner;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.log4j.Appender;
-import org.apache.log4j.BasicConfigurator;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.varia.NullAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
/**
* ExecDriver is the central class in co-ordinating execution of any map-reduce task.
@@ -687,8 +688,10 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
if (noLog) {
// If started from main(), and noLog is on, we should not output
// any logs. To turn the log on, please set -Dtest.silent=false
- BasicConfigurator.resetConfiguration();
- BasicConfigurator.configure(new NullAppender());
+ Logger logger = org.apache.logging.log4j.LogManager.getRootLogger();
+ NullAppender appender = NullAppender.createNullAppender();
+ appender.addToLogger(logger.getName(), Level.ERROR);
+ appender.start();
} else {
setupChildLog4j(conf);
}
@@ -703,10 +706,12 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
// print out the location of the log file for the user so
// that it's easy to find reason for local mode execution failures
- for (Appender appender : Collections.list((Enumeration<Appender>) LogManager.getRootLogger()
- .getAllAppenders())) {
+ for (Appender appender : ((org.apache.logging.log4j.core.Logger) LogManager.getRootLogger())
+ .getAppenders().values()) {
if (appender instanceof FileAppender) {
- console.printInfo("Execution log at: " + ((FileAppender) appender).getFile());
+ console.printInfo("Execution log at: " + ((FileAppender) appender).getFileName());
+ } else if (appender instanceof RollingFileAppender) {
+ console.printInfo("Execution log at: " + ((RollingFileAppender) appender).getFileName());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
index 6a6593c..44dfe3e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
@@ -24,7 +24,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -33,8 +32,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.MapRedStats;
@@ -59,9 +56,11 @@ import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.RunningJob;
import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapred.TaskReport;
-import org.apache.log4j.Appender;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
public class HadoopJobExecHelper {
@@ -492,10 +491,11 @@ public class HadoopJobExecHelper {
sb.append("Logs:\n");
console.printError(sb.toString());
- for (Appender a : Collections.list((Enumeration<Appender>)
- LogManager.getRootLogger().getAllAppenders())) {
- if (a instanceof FileAppender) {
- console.printError((new Path(((FileAppender)a).getFile())).toUri().getPath());
+ for (Appender appender : ((Logger) LogManager.getRootLogger()).getAppenders().values()) {
+ if (appender instanceof FileAppender) {
+ console.printError(((FileAppender) appender).getFileName());
+ } else if (appender instanceof RollingFileAppender) {
+ console.printError(((RollingFileAppender) appender).getFileName());
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
index 3cb9e9c..cee0878 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.hive.ql.io.rcfile.stats;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Enumeration;
import java.util.List;
import org.apache.commons.lang.StringUtils;
@@ -59,9 +57,11 @@ import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
-import org.apache.log4j.Appender;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.LogManager;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Logger;
+import org.apache.logging.log4j.core.appender.FileAppender;
+import org.apache.logging.log4j.core.appender.RollingFileAppender;
/**
* PartialScanTask.
@@ -335,15 +335,15 @@ public class PartialScanTask extends Task<PartialScanWork> implements
// print out the location of the log file for the user so
// that it's easy to find reason for local mode execution failures
- for (Appender appender : Collections
- .list((Enumeration<Appender>) LogManager.getRootLogger()
- .getAllAppenders())) {
+ for (Appender appender : ((Logger) LogManager.getRootLogger()).getAppenders().values()) {
if (appender instanceof FileAppender) {
- console.printInfo("Execution log at: "
- + ((FileAppender) appender).getFile());
+ console.printInfo("Execution log at: " + ((FileAppender) appender).getFileName());
+ } else if (appender instanceof RollingFileAppender) {
+ console.printInfo("Execution log at: " + ((RollingFileAppender) appender).getFileName());
}
}
+
PartialScanWork mergeWork = new PartialScanWork(inputPaths);
DriverContext driverCxt = new DriverContext();
PartialScanTask taskExec = new PartialScanTask();
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
new file mode 100644
index 0000000..46662c4
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicLongArray;
+
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginElement;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * A log4J2 Appender that simply counts logging events in four levels:
+ * fatal, error, warn and info. The class name is used in log4j2.xml
+ */
+@Plugin(name = "HiveEventCounter", category = "Core", elementType = "appender", printObject = true)
+public class HiveEventCounter extends AbstractAppender {
+ private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ private static Configuration configuration = context.getConfiguration();
+ private static final String APPENDER_NAME = "HiveEventCounter";
+ private static final int FATAL = 0;
+ private static final int ERROR = 1;
+ private static final int WARN = 2;
+ private static final int INFO = 3;
+
+ private static class EventCounts {
+ private final AtomicLongArray counts = new AtomicLongArray(4);
+
+ private void incr(int i) {
+ counts.incrementAndGet(i);
+ }
+
+ private long get(int i) {
+ return counts.get(i);
+ }
+ }
+
+ private static EventCounts counts = new EventCounts();
+
+ protected HiveEventCounter(String name, Filter filter,
+ Layout<? extends Serializable> layout, boolean ignoreExceptions) {
+ super(name, filter, layout, ignoreExceptions);
+ }
+
+ @PluginFactory
+ public static HiveEventCounter createInstance(@PluginAttribute("name") String name,
+ @PluginAttribute("ignoreExceptions") boolean ignoreExceptions,
+ @PluginElement("Layout") Layout layout,
+ @PluginElement("Filters") Filter filter) {
+ if (name == null) {
+ name = APPENDER_NAME;
+ }
+
+ if (layout == null) {
+ layout = PatternLayout.createDefaultLayout();
+ }
+ return new HiveEventCounter(name, filter, layout, ignoreExceptions);
+ }
+
+ @InterfaceAudience.Private
+ public static long getFatal() {
+ return counts.get(FATAL);
+ }
+
+ @InterfaceAudience.Private
+ public static long getError() {
+ return counts.get(ERROR);
+ }
+
+ @InterfaceAudience.Private
+ public static long getWarn() {
+ return counts.get(WARN);
+ }
+
+ @InterfaceAudience.Private
+ public static long getInfo() {
+ return counts.get(INFO);
+ }
+
+ @VisibleForTesting
+ public void addToLogger(String loggerName, Level level) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.addAppender(this, level, null);
+ context.updateLoggers();
+ }
+
+ @VisibleForTesting
+ public void removeFromLogger(String loggerName) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.removeAppender(APPENDER_NAME);
+ context.updateLoggers();
+ }
+
+ public void append(LogEvent event) {
+ Level level = event.getLevel();
+ if (level.equals(Level.INFO)) {
+ counts.incr(INFO);
+ } else if (level.equals(Level.WARN)) {
+ counts.incr(WARN);
+ } else if (level.equals(Level.ERROR)) {
+ counts.incr(ERROR);
+ } else if (level.equals(Level.FATAL)) {
+ counts.incr(FATAL);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
new file mode 100644
index 0000000..c4cb7dd
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.io.Serializable;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+/**
+ * A NullAppender merely exists, it never outputs a message to any device.
+ */
+@Plugin(name = "NullAppender", category = "Core", elementType = "appender", printObject = false)
+public class NullAppender extends AbstractAppender {
+
+ private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ private static Configuration configuration = context.getConfiguration();
+
+ protected NullAppender(String name, Filter filter,
+ Layout<? extends Serializable> layout, boolean ignoreExceptions) {
+ super(name, filter, layout, ignoreExceptions);
+ }
+
+ @PluginFactory
+ public static NullAppender createNullAppender() {
+ return new NullAppender("NullAppender", null, PatternLayout.createDefaultLayout(), true);
+ }
+
+ public void addToLogger(String loggerName, Level level) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.addAppender(this, level, null);
+ context.updateLoggers();
+ }
+
+ public void append(LogEvent event) {
+ // no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
deleted file mode 100644
index 6a59d4a..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.log;
-
-import java.lang.management.ManagementFactory;
-import java.lang.management.RuntimeMXBean;
-
-import org.apache.log4j.DailyRollingFileAppender;
-
-public class PidDailyRollingFileAppender extends DailyRollingFileAppender {
-
- @Override
- public void setFile(String file) {
- RuntimeMXBean rt = ManagementFactory.getRuntimeMXBean();
- super.setFile(file + '.' + rt.getName());
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
new file mode 100644
index 0000000..4db10bb
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import java.lang.management.ManagementFactory;
+
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.pattern.AbstractPatternConverter;
+import org.apache.logging.log4j.core.pattern.ArrayPatternConverter;
+import org.apache.logging.log4j.core.pattern.ConverterKeys;
+
+/**
+ * FilePattern converter that converts %pid pattern to <process-id>@<hostname> information
+ * obtained at runtime.
+ *
+ * Example usage:
+ * <RollingFile name="Rolling-default" fileName="test.log" filePattern="test.log.%pid.gz">
+ *
+ * Will generate output file with name containing <process-id>@<hostname> like below
+ * test.log.95232@localhost.gz
+ */
+@Plugin(name = "PidFilePatternConverter", category = "FileConverter")
+@ConverterKeys({ "pid" })
+public class PidFilePatternConverter extends AbstractPatternConverter implements
+ ArrayPatternConverter {
+
+ /**
+ * Private constructor.
+ */
+ private PidFilePatternConverter() {
+ super("pid", "pid");
+ }
+
+ @PluginFactory
+ public static PidFilePatternConverter newInstance() {
+ return new PidFilePatternConverter();
+ }
+
+ public void format(StringBuilder toAppendTo, Object... objects) {
+ toAppendTo.append(ManagementFactory.getRuntimeMXBean().getName());
+ }
+
+ public void format(Object obj, StringBuilder toAppendTo) {
+ toAppendTo.append(ManagementFactory.getRuntimeMXBean().getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/main/resources/hive-exec-log4j.properties
----------------------------------------------------------------------
diff --git a/ql/src/main/resources/hive-exec-log4j.properties b/ql/src/main/resources/hive-exec-log4j.properties
deleted file mode 100644
index 9eaa6b6..0000000
--- a/ql/src/main/resources/hive-exec-log4j.properties
+++ /dev/null
@@ -1,77 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hive.log.threshold=ALL
-hive.root.logger=INFO,FA
-hive.log.dir=${java.io.tmpdir}/${user.name}
-hive.query.id=hadoop
-hive.log.file=${hive.query.id}.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hive.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshhold=${hive.log.threshold}
-
-#
-# File Appender
-#
-
-log4j.appender.FA=org.apache.log4j.FileAppender
-log4j.appender.FA.File=${hive.log.dir}/${hive.log.file}
-log4j.appender.FA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p %c{2}: %m%n
-
-#custom logging levels
-#log4j.logger.xxx=DEBUG
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
-
-
-log4j.category.DataNucleus=ERROR,FA
-log4j.category.Datastore=ERROR,FA
-log4j.category.Datastore.Schema=ERROR,FA
-log4j.category.JPOX.Datastore=ERROR,FA
-log4j.category.JPOX.Plugin=ERROR,FA
-log4j.category.JPOX.MetaData=ERROR,FA
-log4j.category.JPOX.Query=ERROR,FA
-log4j.category.JPOX.General=ERROR,FA
-log4j.category.JPOX.Enhancer=ERROR,FA
-
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/main/resources/hive-exec-log4j2.xml
----------------------------------------------------------------------
diff --git a/ql/src/main/resources/hive-exec-log4j2.xml b/ql/src/main/resources/hive-exec-log4j2.xml
new file mode 100644
index 0000000..c93437c
--- /dev/null
+++ b/ql/src/main/resources/hive-exec-log4j2.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="HiveExecLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="hive.log.threshold">ALL</Property>
+ <Property name="hive.log.level">INFO</Property>
+ <Property name="hive.root.logger">FA</Property>
+ <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
+ <Property name="hive.query.id">hadoop</Property>
+ <Property name="hive.log.file">${sys:hive.query.id}.log</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
+ </Console>
+
+ <!-- Regular File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ </File>
+
+ <!-- Daily Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
+ <!-- <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
+ </Policies>
+ <DefaultRolloverStrategy max="30"/>
+ </RollingFile> -->
+
+ <!-- Size based Rolling File Appender -->
+ <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
+ <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
+ filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
+ <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
+ <Policies>
+ <SizeBasedTriggeringPolicy size="256 MB" />
+ </Policies>
+ <DefaultRolloverStrategy max="10"/>
+ </RollingFile> -->
+
+ <!-- HiveEventCounter appender is loaded from Configuration packages attribute.Sends counts of logging messages at different severity levels to Hadoop Metrics. -->
+ <HiveEventCounter name="EventCounter"/>
+ </Appenders>
+
+ <Loggers>
+ <Root level="${sys:hive.log.threshold}">
+ <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
+ <AppenderRef ref="EventCounter" />
+ </Root>
+
+ <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="DataNucleus" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="Datastore.Schema" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Datastore" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Plugin" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Metadata" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Query" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.General" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ <Logger name="JPOX.Enhancer" level="ERROR">
+ <AppenderRef ref="${sys:hive.root.logger}"/>
+ </Logger>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
new file mode 100644
index 0000000..bdd837e
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.log;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.hive.ql.metadata.StringAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestLog4j2Appenders {
+
+ @Before
+ public void setup() {
+ // programmatically set root logger level to INFO. By default if log4j2-test.xml is not
+ // available root logger will use ERROR log level
+ LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ Configuration config = ctx.getConfiguration();
+ LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
+ loggerConfig.setLevel(Level.INFO);
+ ctx.updateLoggers();
+ }
+
+ @Test
+ public void testStringAppender() throws Exception {
+ // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs
+ Logger logger = LogManager.getRootLogger();
+ // Create a String Appender to capture log output
+ StringAppender appender = StringAppender.createStringAppender("%m");
+ appender.addToLogger(logger.getName(), Level.INFO);
+ appender.start();
+
+ // Log to the string appender
+ logger.info("Hello!");
+ logger.info(" World");
+
+ assertEquals("Hello! World", appender.getOutput());
+ appender.removeFromLogger(LogManager.getRootLogger().getName());
+ }
+
+ @Test
+ public void testHiveEventCounterAppender() throws Exception {
+ Logger logger = LogManager.getRootLogger();
+ HiveEventCounter appender = HiveEventCounter.createInstance("EventCounter", true, null, null);
+ appender.addToLogger(logger.getName(), Level.INFO);
+ appender.start();
+
+ logger.info("Test");
+ logger.info("Test");
+ logger.info("Test");
+ logger.info("Test");
+
+ logger.error("Test");
+ logger.error("Test");
+ logger.error("Test");
+
+ logger.warn("Test");
+ logger.warn("Test");
+
+ logger.fatal("Test");
+
+ // HiveEventCounter will be loaded from hive-log4j2-test.xml before tests are run. The 2 log
+ // info msgs from previous test case will also be counted along with 4 log info msgs in this
+ // test and hence we assert for 6 here
+ assertEquals(6, appender.getInfo());
+ assertEquals(3, appender.getError());
+ assertEquals(2, appender.getWarn());
+ assertEquals(1, appender.getFatal());
+ appender.removeFromLogger(LogManager.getRootLogger().getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
new file mode 100644
index 0000000..17b64d6
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.metadata;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
+import org.apache.logging.log4j.core.appender.OutputStreamManager;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Log4j2 appender that writers to in-memory string object.
+ */
+@Plugin(name = "StringAppender", category = "Core", elementType = "appender", printObject = true)
+public class StringAppender
+ extends AbstractOutputStreamAppender<StringAppender.StringOutputStreamManager> {
+
+ private static final String APPENDER_NAME = "StringAppender";
+ private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ private static Configuration configuration = context.getConfiguration();
+ private StringOutputStreamManager manager;
+
+ /**
+ * Instantiate a WriterAppender and set the output destination to a
+ * new {@link OutputStreamWriter} initialized with <code>os</code>
+ * as its {@link OutputStream}.
+ *
+ * @param name The name of the Appender.
+ * @param layout The layout to format the message.
+ * @param filter
+ * @param ignoreExceptions
+ * @param immediateFlush
+ * @param manager The OutputStreamManager.
+ */
+ protected StringAppender(String name,
+ Layout<? extends Serializable> layout, Filter filter,
+ boolean ignoreExceptions, boolean immediateFlush,
+ StringOutputStreamManager manager) {
+ super(name, layout, filter, ignoreExceptions, immediateFlush, manager);
+ this.manager = manager;
+ }
+
+ @PluginFactory
+ public static StringAppender createStringAppender(
+ @PluginAttribute("name") String nullablePatternString) {
+ PatternLayout layout;
+ if (nullablePatternString == null) {
+ layout = PatternLayout.createDefaultLayout();
+ } else {
+ layout = PatternLayout.createLayout(nullablePatternString, configuration,
+ null, null, true, false, null, null);
+ }
+
+ return new StringAppender(APPENDER_NAME, layout, null, false, true,
+ new StringOutputStreamManager(new ByteArrayOutputStream(), "StringStream", layout));
+ }
+
+ @VisibleForTesting
+ public void addToLogger(String loggerName, Level level) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.addAppender(this, level, null);
+ context.updateLoggers();
+ }
+
+ @VisibleForTesting
+ public void removeFromLogger(String loggerName) {
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
+ loggerConfig.removeAppender(APPENDER_NAME);
+ context.updateLoggers();
+ }
+
+ public String getOutput() {
+ manager.flush();
+ return new String(manager.getStream().toByteArray());
+ }
+
+ public void reset() {
+ manager.reset();
+ }
+
+ protected static class StringOutputStreamManager extends OutputStreamManager {
+ ByteArrayOutputStream stream;
+
+ protected StringOutputStreamManager(ByteArrayOutputStream os, String streamName,
+ Layout<?> layout) {
+ super(os, streamName, layout);
+ stream = os;
+ }
+
+ public ByteArrayOutputStream getStream() {
+ return stream;
+ }
+
+ public void reset() {
+ stream.reset();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
index 99fbd5d..1e2feaa 100755
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.metadata;
import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
-import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -29,8 +28,6 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
-import junit.framework.TestCase;
-
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -56,15 +53,19 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.WriterAppender;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.junit.Assert;
import com.google.common.collect.ImmutableMap;
+import junit.framework.TestCase;
+
/**
* TestHive.
*
@@ -248,36 +249,39 @@ public class TestHive extends TestCase {
* @throws Throwable
*/
public void testMetaStoreApiTiming() throws Throwable {
- // set log level to DEBUG, as this is logged at debug level
- Logger logger = Logger.getLogger("hive.ql.metadata.Hive");
- Level origLevel = logger.getLevel();
- logger.setLevel(Level.DEBUG);
-
- // create an appender to capture the logs in a string
- StringWriter writer = new StringWriter();
- WriterAppender appender = new WriterAppender(new PatternLayout(), writer);
+ // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs
+ Logger logger = LogManager.getLogger("hive.ql.metadata.Hive");
+ Level oldLevel = logger.getLevel();
+ LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ Configuration config = ctx.getConfiguration();
+ LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
+ loggerConfig.setLevel(Level.DEBUG);
+ ctx.updateLoggers();
+
+ // Create a String Appender to capture log output
+ StringAppender appender = StringAppender.createStringAppender("%m");
+ appender.addToLogger(logger.getName(), Level.DEBUG);
+ appender.start();
try {
- logger.addAppender(appender);
-
hm.clearMetaCallTiming();
hm.getAllDatabases();
hm.dumpAndClearMetaCallTiming("test");
- String logStr = writer.toString();
+ String logStr = appender.getOutput();
String expectedString = "getAllDatabases_()=";
Assert.assertTrue(logStr + " should contain <" + expectedString,
logStr.contains(expectedString));
// reset the log buffer, verify new dump without any api call does not contain func
- writer.getBuffer().setLength(0);
+ appender.reset();
hm.dumpAndClearMetaCallTiming("test");
- logStr = writer.toString();
+ logStr = appender.getOutput();
Assert.assertFalse(logStr + " should not contain <" + expectedString,
logStr.contains(expectedString));
-
} finally {
- logger.setLevel(origLevel);
- logger.removeAppender(appender);
+ loggerConfig.setLevel(oldLevel);
+ ctx.updateLoggers();
+ appender.removeFromLogger(logger.getName());
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
index 9d64b10..876ade8 100644
--- a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
+++ b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
@@ -18,9 +18,6 @@
package org.apache.hive.service.cli;
-import org.apache.log4j.Layout;
-import org.apache.log4j.PatternLayout;
-
/**
* CLIServiceUtils.
*
@@ -29,10 +26,6 @@ public class CLIServiceUtils {
private static final char SEARCH_STRING_ESCAPE = '\\';
- public static final Layout verboseLayout = new PatternLayout(
- "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n");
- public static final Layout nonVerboseLayout = new PatternLayout(
- "%-5p : %m%n");
/**
* Convert a SQL search pattern into an equivalent Java Regex.
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
index 70340bd..fb3921f 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
@@ -6,44 +6,94 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hive.service.cli.operation;
-import java.io.CharArrayWriter;
-import java.util.Enumeration;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Serializable;
import java.util.regex.Pattern;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.session.OperationLog;
-import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
-import org.apache.hive.service.cli.CLIServiceUtils;
-import org.apache.log4j.Appender;
-import org.apache.log4j.ConsoleAppender;
-import org.apache.log4j.Layout;
-import org.apache.log4j.Logger;
-import org.apache.log4j.WriterAppender;
-import org.apache.log4j.spi.Filter;
-import org.apache.log4j.spi.LoggingEvent;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
+import org.apache.logging.log4j.core.appender.ConsoleAppender;
+import org.apache.logging.log4j.core.appender.OutputStreamManager;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.filter.AbstractFilter;
+import org.apache.logging.log4j.core.layout.PatternLayout;
import com.google.common.base.Joiner;
/**
- * An Appender to divert logs from individual threads to the LogObject they belong to.
+ * Divert appender to redirect operation logs to separate files.
*/
-public class LogDivertAppender extends WriterAppender {
- private static final Logger LOG = Logger.getLogger(LogDivertAppender.class.getName());
+public class LogDivertAppender
+ extends AbstractOutputStreamAppender<LogDivertAppender.StringOutputStreamManager> {
+ private static final Logger LOG = LogManager.getLogger(LogDivertAppender.class.getName());
+ private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ private static Configuration configuration = context.getConfiguration();
+ public static final Layout<? extends Serializable> verboseLayout = PatternLayout.createLayout(
+ "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n", configuration, null, null, true, false, null, null);
+ public static final Layout<? extends Serializable> nonVerboseLayout = PatternLayout.createLayout(
+ "%-5p : %m%n", configuration, null, null, true, false, null, null);
+
private final OperationManager operationManager;
+ private StringOutputStreamManager manager;
private boolean isVerbose;
- private Layout verboseLayout;
+ private final Layout<? extends Serializable> layout;
+
+ /**
+ * Instantiate a WriterAppender and set the output destination to a
+ * new {@link OutputStreamWriter} initialized with <code>os</code>
+ * as its {@link OutputStream}.
+ *
+ * @param name The name of the Appender.
+ * @param filter Filter
+ * @param manager The OutputStreamManager.
+ * @param operationManager Operation manager
+ */
+ protected LogDivertAppender(String name, Filter filter,
+ StringOutputStreamManager manager, OperationManager operationManager,
+ OperationLog.LoggingLevel loggingMode) {
+ super(name, null, filter, false, true, manager);
+ this.operationManager = operationManager;
+ this.manager = manager;
+ this.isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
+ this.layout = getDefaultLayout();
+ }
+
+ public Layout<? extends Serializable> getDefaultLayout() {
+ // There should be a ConsoleAppender. Copy its Layout.
+ Logger root = LogManager.getRootLogger();
+ Layout layout = null;
+
+ for (Appender ap : ((org.apache.logging.log4j.core.Logger) root).getAppenders().values()) {
+ if (ap.getClass().equals(ConsoleAppender.class)) {
+ layout = ap.getLayout();
+ break;
+ }
+ }
+
+ return layout;
+ }
/**
* A log filter that filters messages coming from the logger with the given names.
@@ -52,31 +102,31 @@ public class LogDivertAppender extends WriterAppender {
* they don't generate more logs for themselves when they process logs.
* White list filter is used for less verbose log collection
*/
- private static class NameFilter extends Filter {
+ private static class NameFilter extends AbstractFilter {
private Pattern namePattern;
- private LoggingLevel loggingMode;
+ private OperationLog.LoggingLevel loggingMode;
private OperationManager operationManager;
/* Patterns that are excluded in verbose logging level.
* Filter out messages coming from log processing classes, or we'll run an infinite loop.
*/
private static final Pattern verboseExcludeNamePattern = Pattern.compile(Joiner.on("|").
- join(new String[] {LOG.getName(), OperationLog.class.getName(),
- OperationManager.class.getName()}));
+ join(new String[]{LOG.getName(), OperationLog.class.getName(),
+ OperationManager.class.getName()}));
/* Patterns that are included in execution logging level.
* In execution mode, show only select logger messages.
*/
private static final Pattern executionIncludeNamePattern = Pattern.compile(Joiner.on("|").
- join(new String[] {"org.apache.hadoop.mapreduce.JobSubmitter",
- "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
- "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));
+ join(new String[]{"org.apache.hadoop.mapreduce.JobSubmitter",
+ "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
+ "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));
/* Patterns that are included in performance logging level.
* In performance mode, show execution and performance logger messages.
*/
private static final Pattern performanceIncludeNamePattern = Pattern.compile(
- executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());
+ executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());
private void setCurrentNamePattern(OperationLog.LoggingLevel mode) {
if (mode == OperationLog.LoggingLevel.VERBOSE) {
@@ -88,26 +138,25 @@ public class LogDivertAppender extends WriterAppender {
}
}
- public NameFilter(
- OperationLog.LoggingLevel loggingMode, OperationManager op) {
+ public NameFilter(OperationLog.LoggingLevel loggingMode, OperationManager op) {
this.operationManager = op;
this.loggingMode = loggingMode;
setCurrentNamePattern(loggingMode);
}
@Override
- public int decide(LoggingEvent ev) {
+ public Result filter(LogEvent event) {
OperationLog log = operationManager.getOperationLogByThread();
boolean excludeMatches = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
if (log == null) {
- return Filter.DENY;
+ return Result.DENY;
}
OperationLog.LoggingLevel currentLoggingMode = log.getOpLoggingLevel();
// If logging is disabled, deny everything.
if (currentLoggingMode == OperationLog.LoggingLevel.NONE) {
- return Filter.DENY;
+ return Result.DENY;
}
// Look at the current session's setting
// and set the pattern and excludeMatches accordingly.
@@ -116,88 +165,58 @@ public class LogDivertAppender extends WriterAppender {
setCurrentNamePattern(loggingMode);
}
- boolean isMatch = namePattern.matcher(ev.getLoggerName()).matches();
+ boolean isMatch = namePattern.matcher(event.getLoggerName()).matches();
if (excludeMatches == isMatch) {
// Deny if this is black-list filter (excludeMatches = true) and it
- // matched
- // or if this is whitelist filter and it didn't match
- return Filter.DENY;
+ // matched or if this is whitelist filter and it didn't match
+ return Result.DENY;
}
- return Filter.NEUTRAL;
+ return Result.NEUTRAL;
}
}
- /** This is where the log message will go to */
- private final CharArrayWriter writer = new CharArrayWriter();
-
- private void setLayout (boolean isVerbose, Layout lo) {
- if (isVerbose) {
- if (lo == null) {
- lo = CLIServiceUtils.verboseLayout;
- LOG.info("Cannot find a Layout from a ConsoleAppender. Using default Layout pattern.");
- }
- } else {
- lo = CLIServiceUtils.nonVerboseLayout;
- }
- setLayout(lo);
+ public static LogDivertAppender createInstance(OperationManager operationManager,
+ OperationLog.LoggingLevel loggingMode) {
+ return new LogDivertAppender("LogDivertAppender", new NameFilter(loggingMode, operationManager),
+ new StringOutputStreamManager(new ByteArrayOutputStream(), "StringStream", null),
+ operationManager, loggingMode);
}
- private void initLayout(boolean isVerbose) {
- // There should be a ConsoleAppender. Copy its Layout.
- Logger root = Logger.getRootLogger();
- Layout layout = null;
-
- Enumeration<?> appenders = root.getAllAppenders();
- while (appenders.hasMoreElements()) {
- Appender ap = (Appender) appenders.nextElement();
- if (ap.getClass().equals(ConsoleAppender.class)) {
- layout = ap.getLayout();
- break;
- }
- }
- setLayout(isVerbose, layout);
+ public String getOutput() {
+ return new String(manager.getStream().toByteArray());
}
- public LogDivertAppender(OperationManager operationManager,
- OperationLog.LoggingLevel loggingMode) {
- isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
- initLayout(isVerbose);
- setWriter(writer);
- setName("LogDivertAppender");
- this.operationManager = operationManager;
- this.verboseLayout = isVerbose ? layout : CLIServiceUtils.verboseLayout;
- addFilter(new NameFilter(loggingMode, operationManager));
+ @Override
+ public void start() {
+ super.start();
}
@Override
- public void doAppend(LoggingEvent event) {
- OperationLog log = operationManager.getOperationLogByThread();
+ public Layout<? extends Serializable> getLayout() {
- // Set current layout depending on the verbose/non-verbose mode.
+ // If there is a logging level change from verbose->non-verbose or vice-versa since
+ // the last subAppend call, change the layout to preserve consistency.
+ OperationLog log = operationManager.getOperationLogByThread();
if (log != null) {
- boolean isCurrModeVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
+ isVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
+ }
- // If there is a logging level change from verbose->non-verbose or vice-versa since
- // the last subAppend call, change the layout to preserve consistency.
- if (isCurrModeVerbose != isVerbose) {
- isVerbose = isCurrModeVerbose;
- setLayout(isVerbose, verboseLayout);
- }
+ // layout is immutable in log4j2, so we cheat here and return a different layout when
+ // verbosity changes
+ if (isVerbose) {
+ return verboseLayout;
+ } else {
+ return layout == null ? nonVerboseLayout : layout;
}
- super.doAppend(event);
}
- /**
- * Overrides WriterAppender.subAppend(), which does the real logging. No need
- * to worry about concurrency since log4j calls this synchronously.
- */
@Override
- protected void subAppend(LoggingEvent event) {
- super.subAppend(event);
- // That should've gone into our writer. Notify the LogContext.
- String logOutput = writer.toString();
- writer.reset();
+ public void append(LogEvent event) {
+ super.append(event);
+
+ String logOutput = getOutput();
+ manager.reset();
OperationLog log = operationManager.getOperationLogByThread();
if (log == null) {
@@ -206,4 +225,22 @@ public class LogDivertAppender extends WriterAppender {
}
log.writeOperationLog(logOutput);
}
+
+ protected static class StringOutputStreamManager extends OutputStreamManager {
+ ByteArrayOutputStream stream;
+
+ protected StringOutputStreamManager(ByteArrayOutputStream os, String streamName,
+ Layout<?> layout) {
+ super(os, streamName, layout);
+ stream = os;
+ }
+
+ public ByteArrayOutputStream getStream() {
+ return stream;
+ }
+
+ public void reset() {
+ stream.reset();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
index 9b0a519..304a525 100644
--- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
+++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
@@ -41,8 +41,11 @@ import org.apache.hive.service.cli.RowSet;
import org.apache.hive.service.cli.RowSetFactory;
import org.apache.hive.service.cli.TableSchema;
import org.apache.hive.service.cli.session.HiveSession;
-import org.apache.log4j.Appender;
-import org.apache.log4j.Logger;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
+import org.apache.logging.log4j.core.config.LoggerConfig;
/**
* OperationManager.
@@ -50,7 +53,6 @@ import org.apache.log4j.Logger;
*/
public class OperationManager extends AbstractService {
private final Log LOG = LogFactory.getLog(OperationManager.class.getName());
-
private final Map<OperationHandle, Operation> handleToOperation =
new HashMap<OperationHandle, Operation>();
@@ -83,8 +85,13 @@ public class OperationManager extends AbstractService {
private void initOperationLogCapture(String loggingMode) {
// Register another Appender (with the same layout) that talks to us.
- Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode));
- Logger.getRootLogger().addAppender(ap);
+ Appender ap = LogDivertAppender.createInstance(this, OperationLog.getLoggingLevel(loggingMode));
+ LoggerContext context = (LoggerContext) LogManager.getContext(false);
+ Configuration configuration = context.getConfiguration();
+ LoggerConfig loggerConfig = configuration.getLoggerConfig(LogManager.getLogger().getName());
+ loggerConfig.addAppender(ap, null, null);
+ context.updateLoggers();
+ ap.start();
}
public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/shims/common/pom.xml
----------------------------------------------------------------------
diff --git a/shims/common/pom.xml b/shims/common/pom.xml
index 9e9a3b7..dfdec2b 100644
--- a/shims/common/pom.xml
+++ b/shims/common/pom.xml
@@ -41,14 +41,19 @@
<version>${commons-logging.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
- <groupId>log4j</groupId>
- <artifactId>apache-log4j-extras</artifactId>
- <version>${log4j-extras.version}</version>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
----------------------------------------------------------------------
diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
deleted file mode 100644
index 224b135..0000000
--- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.shims;
-
-import org.apache.log4j.Appender;
-import org.apache.log4j.AppenderSkeleton;
-import org.apache.log4j.Layout;
-import org.apache.log4j.spi.ErrorHandler;
-import org.apache.log4j.spi.Filter;
-import org.apache.log4j.spi.LoggingEvent;
-import org.apache.log4j.spi.OptionHandler;
-
-public class HiveEventCounter implements Appender, OptionHandler {
-
- AppenderSkeleton hadoopEventCounter;
-
- public HiveEventCounter() {
- hadoopEventCounter = ShimLoader.getEventCounter();
- }
-
- @Override
- public void close() {
- hadoopEventCounter.close();
- }
-
- @Override
- public boolean requiresLayout() {
- return hadoopEventCounter.requiresLayout();
- }
-
- @Override
- public void addFilter(Filter filter) {
- hadoopEventCounter.addFilter(filter);
- }
-
- @Override
- public void clearFilters() {
- hadoopEventCounter.clearFilters();
- }
-
- @Override
- public void doAppend(LoggingEvent event) {
- hadoopEventCounter.doAppend(event);
- }
-
- @Override
- public ErrorHandler getErrorHandler() {
- return hadoopEventCounter.getErrorHandler();
- }
-
- @Override
- public Filter getFilter() {
- return hadoopEventCounter.getFilter();
- }
-
- @Override
- public Layout getLayout() {
- return hadoopEventCounter.getLayout();
- }
-
- @Override
- public String getName() {
- return hadoopEventCounter.getName();
- }
-
- @Override
- public void setErrorHandler(ErrorHandler handler) {
- hadoopEventCounter.setErrorHandler(handler);
- }
-
- @Override
- public void setLayout(Layout layout) {
- hadoopEventCounter.setLayout(layout);
- }
-
- @Override
- public void setName(String name) {
- hadoopEventCounter.setName(name);
- }
-
- @Override
- public void activateOptions() {
- hadoopEventCounter.activateOptions();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/spark-client/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/spark-client/src/test/resources/log4j.properties b/spark-client/src/test/resources/log4j.properties
deleted file mode 100644
index 93a60cc..0000000
--- a/spark-client/src/test/resources/log4j.properties
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file target/unit-tests.log
-log4j.rootCategory=DEBUG, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/spark-client/src/test/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/spark-client/src/test/resources/log4j2.xml b/spark-client/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..a435069
--- /dev/null
+++ b/spark-client/src/test/resources/log4j2.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Configuration status="info" strict="true" name="SparkClientLog4j2"
+ packages="org.apache.hadoop.hive.ql.log">
+
+ <Properties>
+ <Property name="spark.log.level">DEBUG</Property>
+ <Property name="spark.root.logger">console</Property>
+ </Properties>
+
+ <Appenders>
+ <Console name="console" target="SYSTEM_ERR">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n"/>
+ </Console>
+ </Appenders>
+
+ <Loggers>
+ <Root level="DEBUG">
+ <AppenderRef ref="${sys:spark.root.logger}" level="${sys:spark.log.level}"/>
+ </Root>
+ </Loggers>
+
+</Configuration>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 71b51b8..71b79f1 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -32,13 +32,6 @@
</properties>
<dependencies>
- <!-- dependencies are always listed in sorted order by groupId, artifectId -->
- <!-- inter-project -->
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <version>${log4j.version}</version>
- </dependency>
<!-- test inter-project -->
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/pom.xml
----------------------------------------------------------------------
diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
index 211678e..2cf7f45 100644
--- a/testutils/ptest2/pom.xml
+++ b/testutils/ptest2/pom.xml
@@ -64,6 +64,26 @@ limitations under the License.
<version>15.0</version>
</dependency>
<dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-web</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-jcl</artifactId>
+ <version>${log4j2.version}</version>
+ </dependency>
+ <dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/log4j.properties b/testutils/ptest2/src/main/resources/log4j.properties
deleted file mode 100644
index edb9696..0000000
--- a/testutils/ptest2/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,37 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-hive.ptest.logdir=target
-
-log4j.rootLogger=DEBUG,FILE
-log4j.threshhold=ALL
-
-log4j.appender.FILE=org.apache.log4j.RollingFileAppender
-log4j.appender.FILE.File=${hive.ptest.logdir}/ptest.log
-log4j.appender.FILE.MaxFileSize=50MB
-log4j.appender.FILE.MaxBackupIndex=1
-log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
-log4j.appender.FILE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-
-log4j.logger.org.apache.http=INFO
-log4j.logger.org.springframework=INFO
-log4j.logger.org.jclouds=INFO
-log4j.logger.jclouds=INFO
-log4j.logger.org.apache.hive=DEBUG
-log4j.logger.org.apache.http=TRACE
-
-# Silence useless ZK logs
-log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN
-log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN