You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/02/20 21:51:48 UTC
svn commit: r1661217 - in /hive/branches/llap:
llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/
llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/
ql/src/java/org/apache/hadoop/hive/llap/
ql/src/java/org/apache/hadoop/hive/ql/io/...
Author: sershe
Date: Fri Feb 20 20:51:47 2015
New Revision: 1661217
URL: http://svn.apache.org/r1661217
Log:
HIVE-9730p1 : LLAP: make sure logging is never called when not needed (in LLAP proper)
Added:
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java
Modified:
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapInputFormat.java Fri Feb 20 20:51:47 2015
@@ -23,8 +23,6 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.llap.Consumer;
import org.apache.hadoop.hive.llap.ConsumerFeedback;
import org.apache.hadoop.hive.llap.DebugUtils;
@@ -48,7 +46,6 @@ import org.apache.hadoop.mapred.Reporter
public class LlapInputFormat
implements InputFormat<NullWritable, VectorizedRowBatch>, VectorizedInputFormatInterface {
- private static final Log LOG = LogFactory.getLog(LlapInputFormat.class);
private final LlapIoImpl llapIo;
private InputFormat sourceInputFormat;
@@ -65,7 +62,7 @@ public class LlapInputFormat
InputSplit split, JobConf job, Reporter reporter) throws IOException {
boolean isVectorMode = Utilities.isVectorMode(job);
if (!isVectorMode) {
- LOG.error("No llap in non-vectorized mode");
+ LlapIoImpl.LOG.error("No llap in non-vectorized mode");
throw new UnsupportedOperationException("No llap in non-vectorized mode");
}
FileSplit fileSplit = (FileSplit)split;
@@ -160,7 +157,7 @@ public class LlapInputFormat
// We are waiting for next block. Either we will get it, or be told we are done.
boolean doLogBlocking = DebugUtils.isTraceMttEnabled() && isNothingToReport();
if (doLogBlocking) {
- LlapIoImpl.LOG.info("next will block"); // TODO: separate log objects
+ LlapIoImpl.LOG.info("next will block");
}
while (isNothingToReport()) {
pendingData.wait(100);
@@ -202,7 +199,7 @@ public class LlapInputFormat
@Override
public void close() throws IOException {
- if (DebugUtils.isTraceEnabled()) {
+ if (DebugUtils.isTraceMttEnabled()) {
LlapIoImpl.LOG.info("close called; closed " + isClosed + ", done " + isDone
+ ", err " + pendingError + ", pending " + pendingData.size());
}
@@ -220,8 +217,8 @@ public class LlapInputFormat
@Override
public void setDone() {
- if (DebugUtils.isTraceEnabled()) {
- LlapIoImpl.LOG.info("setDone called; cclosed " + isClosed
+ if (DebugUtils.isTraceMttEnabled()) {
+ LlapIoImpl.LOG.info("setDone called; closed " + isClosed
+ ", done " + isDone + ", err " + pendingError + ", pending " + pendingData.size());
}
synchronized (pendingData) {
@@ -232,7 +229,7 @@ public class LlapInputFormat
@Override
public void consumeData(ColumnVectorBatch data) {
- if (DebugUtils.isTraceEnabled()) {
+ if (DebugUtils.isTraceMttEnabled()) {
LlapIoImpl.LOG.info("consume called; closed " + isClosed + ", done " + isDone
+ ", err " + pendingError + ", pending " + pendingData.size());
}
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java Fri Feb 20 20:51:47 2015
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.llap.LogLevels;
import org.apache.hadoop.hive.llap.cache.Allocator;
import org.apache.hadoop.hive.llap.cache.BuddyAllocator;
import org.apache.hadoop.hive.llap.cache.Cache;
@@ -48,6 +49,7 @@ import org.apache.hadoop.mapred.InputSpl
public class LlapIoImpl implements LlapIo<VectorizedRowBatch> {
public static final Log LOG = LogFactory.getLog(LlapIoImpl.class);
+ public static final LogLevels LOGL = new LogLevels(LOG);
private final OrcColumnVectorProducer cvp;
private final OrcEncodedDataProducer edp;
Modified: hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java (original)
+++ hive/branches/llap/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataProducer.java Fri Feb 20 20:51:47 2015
@@ -114,7 +114,9 @@ public class OrcEncodedDataProducer impl
@Override
public Void call() throws IOException {
- LlapIoImpl.LOG.info("Processing split for " + internedFilePath);
+ if (LlapIoImpl.LOGL.isInfoEnabled()) {
+ LlapIoImpl.LOG.info("Processing split for " + internedFilePath);
+ }
if (isStopped) return null;
orcReader = null;
// 1. Get FILE metadata from cache, or create the reader and read it.
@@ -457,7 +459,7 @@ public class OrcEncodedDataProducer impl
long offset = split.getStart(), maxOffset = offset + split.getLength();
stripeIxFrom = -1;
int stripeIxTo = -1;
- if (LlapIoImpl.LOG.isDebugEnabled()) {
+ if (LlapIoImpl.LOGL.isDebugEnabled()) {
String tmp = "FileSplit {" + split.getStart() + ", " + split.getLength() + "}; stripes ";
for (StripeInformation stripe : stripes) {
tmp += "{" + stripe.getOffset() + ", " + stripe.getLength() + "}, ";
Added: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java?rev=1661217&view=auto
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java (added)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/llap/LogLevels.java Fri Feb 20 20:51:47 2015
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional debugrmation
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.llap;
+
+import org.apache.commons.logging.Log;
+
+public class LogLevels {
+ private final boolean isT, isD, isI, isW, isE;
+
+ public LogLevels(Log log) {
+ isT = log.isTraceEnabled();
+ isD = log.isDebugEnabled();
+ isI = log.isInfoEnabled();
+ isW = log.isWarnEnabled();
+ isE = log.isErrorEnabled();
+ }
+
+ public boolean isTraceEnabled() {
+ return isT;
+ }
+
+ public boolean isDebugEnabled() {
+ return isD;
+ }
+
+ public boolean isInfoEnabled() {
+ return isI;
+ }
+
+ public boolean isWarnEnabled() {
+ return isW;
+ }
+
+ public boolean isErrorEnabled() {
+ return isE;
+ }
+}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/InStream.java Fri Feb 20 20:51:47 2015
@@ -28,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.DiskRange;
import org.apache.hadoop.hive.llap.DebugUtils;
+import org.apache.hadoop.hive.llap.LogLevels;
import org.apache.hadoop.hive.llap.io.api.EncodedColumnBatch.StreamBuffer;
import org.apache.hadoop.hive.llap.io.api.cache.LlapMemoryBuffer;
import org.apache.hadoop.hive.llap.io.api.cache.LowLevelCache;
@@ -40,6 +41,8 @@ import com.google.common.annotations.Vis
public abstract class InStream extends InputStream {
private static final Log LOG = LogFactory.getLog(InStream.class);
+ private static final LogLevels LOGL = new LogLevels(LOG);
+
protected final String fileName;
protected final String name;
protected final long length;
@@ -463,7 +466,7 @@ public abstract class InStream extends I
public abstract void seek(PositionProvider index) throws IOException;
private static void logEmptySeek(String name) {
- if (LOG.isWarnEnabled()) {
+ if (LOGL.isWarnEnabled()) {
LOG.warn("Attempting seek into empty stream (" + name + ") Skipping stream.");
}
}
Modified: hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java?rev=1661217&r1=1661216&r2=1661217&view=diff
==============================================================================
--- hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java (original)
+++ hive/branches/llap/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LlapUtils.java Fri Feb 20 20:51:47 2015
@@ -19,12 +19,9 @@ package org.apache.hadoop.hive.ql.io.orc
import java.io.IOException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.io.orc.BitFieldReader;
public class LlapUtils {
- public static final Log LOG = LogFactory.getLog(LlapUtils.class);
public static final int DOUBLE_GROUP_SIZE = 64; // just happens to be equal to bitmask size
/** Helper for readPresentStream. */